{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!coding:utf-8\n",
    "#Load the librarys\n",
    "import pandas as pd #To work with dataset\n",
    "import numpy as np #Math library\n",
    "import seaborn as sns #Graph library that use matplot in background\n",
    "import matplotlib.pyplot as plt #to plot some parameters in seaborn\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score # to split the data\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, fbeta_score #To evaluate our model\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Algorithmns models to be compared\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "#Importing the data\n",
    "df_credit = pd.read_csv(\"./input/german_credit_data.csv\",index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Job</th>\n",
       "      <th>Housing</th>\n",
       "      <th>Saving accounts</th>\n",
       "      <th>Checking account</th>\n",
       "      <th>Credit amount</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Purpose</th>\n",
       "      <th>Risk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>67</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>own</td>\n",
       "      <td>NaN</td>\n",
       "      <td>little</td>\n",
       "      <td>1169</td>\n",
       "      <td>6</td>\n",
       "      <td>radio/TV</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22</td>\n",
       "      <td>female</td>\n",
       "      <td>2</td>\n",
       "      <td>own</td>\n",
       "      <td>little</td>\n",
       "      <td>moderate</td>\n",
       "      <td>5951</td>\n",
       "      <td>48</td>\n",
       "      <td>radio/TV</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>own</td>\n",
       "      <td>little</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2096</td>\n",
       "      <td>12</td>\n",
       "      <td>education</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>free</td>\n",
       "      <td>little</td>\n",
       "      <td>little</td>\n",
       "      <td>7882</td>\n",
       "      <td>42</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>free</td>\n",
       "      <td>little</td>\n",
       "      <td>little</td>\n",
       "      <td>4870</td>\n",
       "      <td>24</td>\n",
       "      <td>car</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>35</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>free</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9055</td>\n",
       "      <td>36</td>\n",
       "      <td>education</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>53</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>own</td>\n",
       "      <td>quite rich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2835</td>\n",
       "      <td>24</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>35</td>\n",
       "      <td>male</td>\n",
       "      <td>3</td>\n",
       "      <td>rent</td>\n",
       "      <td>little</td>\n",
       "      <td>moderate</td>\n",
       "      <td>6948</td>\n",
       "      <td>36</td>\n",
       "      <td>car</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>61</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>own</td>\n",
       "      <td>rich</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3059</td>\n",
       "      <td>12</td>\n",
       "      <td>radio/TV</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>male</td>\n",
       "      <td>3</td>\n",
       "      <td>own</td>\n",
       "      <td>little</td>\n",
       "      <td>moderate</td>\n",
       "      <td>5234</td>\n",
       "      <td>30</td>\n",
       "      <td>car</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age     Sex  Job Housing Saving accounts Checking account  Credit amount  \\\n",
       "0   67    male    2     own             NaN           little           1169   \n",
       "1   22  female    2     own          little         moderate           5951   \n",
       "2   49    male    1     own          little              NaN           2096   \n",
       "3   45    male    2    free          little           little           7882   \n",
       "4   53    male    2    free          little           little           4870   \n",
       "5   35    male    1    free             NaN              NaN           9055   \n",
       "6   53    male    2     own      quite rich              NaN           2835   \n",
       "7   35    male    3    rent          little         moderate           6948   \n",
       "8   61    male    1     own            rich              NaN           3059   \n",
       "9   28    male    3     own          little         moderate           5234   \n",
       "\n",
       "   Duration              Purpose  Risk  \n",
       "0         6             radio/TV  good  \n",
       "1        48             radio/TV   bad  \n",
       "2        12            education  good  \n",
       "3        42  furniture/equipment  good  \n",
       "4        24                  car   bad  \n",
       "5        36            education  good  \n",
       "6        24  furniture/equipment  good  \n",
       "7        36                  car  good  \n",
       "8        12             radio/TV  good  \n",
       "9        30                  car   bad  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_credit[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1000 entries, 0 to 999\n",
      "Data columns (total 10 columns):\n",
      "Age                 1000 non-null int64\n",
      "Sex                 1000 non-null object\n",
      "Job                 1000 non-null int64\n",
      "Housing             1000 non-null object\n",
      "Saving accounts     817 non-null object\n",
      "Checking account    606 non-null object\n",
      "Credit amount       1000 non-null int64\n",
      "Duration            1000 non-null int64\n",
      "Purpose             1000 non-null object\n",
      "Risk                1000 non-null object\n",
      "dtypes: int64(4), object(6)\n",
      "memory usage: 85.9+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(df_credit.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot_encoder(df, nan_as_category = False):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category, drop_first=True)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "interval = (18, 25, 35, 60, 120)\n",
    "\n",
    "cats = ['Student', 'Young', 'Adult', 'Senior']\n",
    "df_credit[\"Age_cat\"] = pd.cut(df_credit.Age, interval, labels=cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_credit['Saving accounts'] = df_credit['Saving accounts'].fillna('no_inf')\n",
    "df_credit['Checking account'] = df_credit['Checking account'].fillna('no_inf')\n",
    "\n",
    "#Purpose to Dummies Variable\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit.Purpose, drop_first=True, prefix='Purpose'), left_index=True, right_index=True)\n",
    "#Sex feature in dummies\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit.Sex, drop_first=True, prefix='Sex'), left_index=True, right_index=True)\n",
    "# Housing get dummies\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit.Housing, drop_first=True, prefix='Housing'), left_index=True, right_index=True)\n",
    "# Housing get Saving Accounts\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit[\"Saving accounts\"], drop_first=True, prefix='Savings'), left_index=True, right_index=True)\n",
    "# Housing get Risk\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit.Risk, prefix='Risk'), left_index=True, right_index=True)\n",
    "# Housing get Checking Account\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit[\"Checking account\"], drop_first=True, prefix='Check'), left_index=True, right_index=True)\n",
    "# Housing get Age categorical\n",
    "df_credit = df_credit.merge(pd.get_dummies(df_credit[\"Age_cat\"], drop_first=True, prefix='Age_cat'), left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Excluding the missing columns\n",
    "del df_credit[\"Saving accounts\"]\n",
    "del df_credit[\"Checking account\"]\n",
    "del df_credit[\"Purpose\"]\n",
    "del df_credit[\"Sex\"]\n",
    "del df_credit[\"Housing\"]\n",
    "del df_credit[\"Age_cat\"]\n",
    "del df_credit[\"Risk\"]\n",
    "del df_credit['Risk_good']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_credit['Credit amount'] = np.log(df_credit['Credit amount'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df_credit.drop('Risk_bad', 1).values\n",
    "y = df_credit[\"Risk_bad\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Spliting X and y into train and test version\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "kf = KFold(n_splits=10, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"booster\":\"gbtree\",\n",
    "    \"objective\":\"binary:logistic\",\n",
    "    \"eval_metiic\":\"logloss\",\n",
    "    \"eta\":0.1,\n",
    "    \"max_depth\":10,\n",
    "    \"missing\":0,\n",
    "    \"seed\":0,\n",
    "    \"silent\":1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "auc_score = []\n",
    "acc_score = []\n",
    "recall_score = []\n",
    "prec_score = []\n",
    "f1_score = []\n",
    "beta_score = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\teval-error:0.36\ttrain-error:0.138889\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.32\ttrain-error:0.132222\n",
      "[2]\teval-error:0.31\ttrain-error:0.123333\n",
      "[3]\teval-error:0.27\ttrain-error:0.116667\n",
      "[4]\teval-error:0.26\ttrain-error:0.113333\n",
      "[5]\teval-error:0.26\ttrain-error:0.115556\n",
      "[6]\teval-error:0.25\ttrain-error:0.11\n",
      "[7]\teval-error:0.26\ttrain-error:0.11\n",
      "[8]\teval-error:0.25\ttrain-error:0.106667\n",
      "[9]\teval-error:0.25\ttrain-error:0.104444\n",
      "[10]\teval-error:0.25\ttrain-error:0.106667\n",
      "[11]\teval-error:0.24\ttrain-error:0.101111\n",
      "[12]\teval-error:0.25\ttrain-error:0.097778\n",
      "[13]\teval-error:0.24\ttrain-error:0.09\n",
      "[14]\teval-error:0.24\ttrain-error:0.086667\n",
      "[15]\teval-error:0.24\ttrain-error:0.08\n",
      "[16]\teval-error:0.24\ttrain-error:0.08\n",
      "[17]\teval-error:0.24\ttrain-error:0.077778\n",
      "[18]\teval-error:0.22\ttrain-error:0.074444\n",
      "[19]\teval-error:0.22\ttrain-error:0.074444\n",
      "[20]\teval-error:0.22\ttrain-error:0.068889\n",
      "[21]\teval-error:0.22\ttrain-error:0.065556\n",
      "[22]\teval-error:0.21\ttrain-error:0.066667\n",
      "[23]\teval-error:0.22\ttrain-error:0.065556\n",
      "[24]\teval-error:0.22\ttrain-error:0.063333\n",
      "[25]\teval-error:0.22\ttrain-error:0.062222\n",
      "[26]\teval-error:0.22\ttrain-error:0.057778\n",
      "[27]\teval-error:0.21\ttrain-error:0.055556\n",
      "[28]\teval-error:0.21\ttrain-error:0.054444\n",
      "[29]\teval-error:0.21\ttrain-error:0.051111\n",
      "[30]\teval-error:0.21\ttrain-error:0.048889\n",
      "[31]\teval-error:0.21\ttrain-error:0.047778\n",
      "[32]\teval-error:0.22\ttrain-error:0.048889\n",
      "[33]\teval-error:0.22\ttrain-error:0.041111\n",
      "[34]\teval-error:0.21\ttrain-error:0.042222\n",
      "[35]\teval-error:0.21\ttrain-error:0.038889\n",
      "[36]\teval-error:0.21\ttrain-error:0.037778\n",
      "[37]\teval-error:0.21\ttrain-error:0.037778\n",
      "[38]\teval-error:0.21\ttrain-error:0.037778\n",
      "[39]\teval-error:0.22\ttrain-error:0.04\n",
      "[40]\teval-error:0.22\ttrain-error:0.035556\n",
      "[41]\teval-error:0.22\ttrain-error:0.035556\n",
      "[42]\teval-error:0.22\ttrain-error:0.034444\n",
      "[43]\teval-error:0.22\ttrain-error:0.034444\n",
      "[44]\teval-error:0.22\ttrain-error:0.03\n",
      "[45]\teval-error:0.23\ttrain-error:0.028889\n",
      "[46]\teval-error:0.22\ttrain-error:0.026667\n",
      "[47]\teval-error:0.23\ttrain-error:0.025556\n",
      "[48]\teval-error:0.23\ttrain-error:0.025556\n",
      "[49]\teval-error:0.23\ttrain-error:0.025556\n",
      "[50]\teval-error:0.22\ttrain-error:0.025556\n",
      "[51]\teval-error:0.22\ttrain-error:0.023333\n",
      "[52]\teval-error:0.22\ttrain-error:0.023333\n",
      "[53]\teval-error:0.22\ttrain-error:0.022222\n",
      "[54]\teval-error:0.23\ttrain-error:0.02\n",
      "[55]\teval-error:0.23\ttrain-error:0.018889\n",
      "[56]\teval-error:0.23\ttrain-error:0.015556\n",
      "[57]\teval-error:0.23\ttrain-error:0.014444\n",
      "[58]\teval-error:0.23\ttrain-error:0.014444\n",
      "[59]\teval-error:0.24\ttrain-error:0.013333\n",
      "[60]\teval-error:0.24\ttrain-error:0.012222\n",
      "[61]\teval-error:0.24\ttrain-error:0.011111\n",
      "[62]\teval-error:0.24\ttrain-error:0.01\n",
      "[63]\teval-error:0.24\ttrain-error:0.01\n",
      "[64]\teval-error:0.24\ttrain-error:0.01\n",
      "[65]\teval-error:0.24\ttrain-error:0.01\n",
      "[66]\teval-error:0.24\ttrain-error:0.01\n",
      "[67]\teval-error:0.24\ttrain-error:0.008889\n",
      "[68]\teval-error:0.23\ttrain-error:0.008889\n",
      "[69]\teval-error:0.23\ttrain-error:0.008889\n",
      "[70]\teval-error:0.24\ttrain-error:0.008889\n",
      "[71]\teval-error:0.24\ttrain-error:0.007778\n",
      "[72]\teval-error:0.24\ttrain-error:0.007778\n",
      "[73]\teval-error:0.24\ttrain-error:0.007778\n",
      "[74]\teval-error:0.23\ttrain-error:0.007778\n",
      "[75]\teval-error:0.23\ttrain-error:0.007778\n",
      "[76]\teval-error:0.23\ttrain-error:0.007778\n",
      "[77]\teval-error:0.23\ttrain-error:0.006667\n",
      "[78]\teval-error:0.23\ttrain-error:0.006667\n",
      "[79]\teval-error:0.22\ttrain-error:0.006667\n",
      "[80]\teval-error:0.22\ttrain-error:0.005556\n",
      "[81]\teval-error:0.22\ttrain-error:0.005556\n",
      "[82]\teval-error:0.22\ttrain-error:0.004444\n",
      "[83]\teval-error:0.22\ttrain-error:0.004444\n",
      "[84]\teval-error:0.22\ttrain-error:0.003333\n",
      "[85]\teval-error:0.22\ttrain-error:0.003333\n",
      "[86]\teval-error:0.22\ttrain-error:0.003333\n",
      "[87]\teval-error:0.22\ttrain-error:0.003333\n",
      "[88]\teval-error:0.22\ttrain-error:0.003333\n",
      "[89]\teval-error:0.22\ttrain-error:0.003333\n",
      "[90]\teval-error:0.22\ttrain-error:0.003333\n",
      "[91]\teval-error:0.22\ttrain-error:0.002222\n",
      "[92]\teval-error:0.22\ttrain-error:0.002222\n",
      "[93]\teval-error:0.22\ttrain-error:0.002222\n",
      "[94]\teval-error:0.22\ttrain-error:0.002222\n",
      "[95]\teval-error:0.22\ttrain-error:0.002222\n",
      "[96]\teval-error:0.22\ttrain-error:0.002222\n",
      "[97]\teval-error:0.22\ttrain-error:0.002222\n",
      "[98]\teval-error:0.22\ttrain-error:0.002222\n",
      "[99]\teval-error:0.22\ttrain-error:0.002222\n",
      "[100]\teval-error:0.21\ttrain-error:0.002222\n",
      "[101]\teval-error:0.22\ttrain-error:0.002222\n",
      "Stopping. Best iteration:\n",
      "[91]\teval-error:0.22\ttrain-error:0.002222\n",
      "\n",
      "[1 0 0 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 1 0 1 0 1 0\n",
      " 1 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 1 0 0 1]\n",
      "[0]\teval-error:0.31\ttrain-error:0.145556\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.32\ttrain-error:0.114444\n",
      "[2]\teval-error:0.31\ttrain-error:0.123333\n",
      "[3]\teval-error:0.27\ttrain-error:0.125556\n",
      "[4]\teval-error:0.28\ttrain-error:0.123333\n",
      "[5]\teval-error:0.28\ttrain-error:0.121111\n",
      "[6]\teval-error:0.3\ttrain-error:0.117778\n",
      "[7]\teval-error:0.29\ttrain-error:0.102222\n",
      "[8]\teval-error:0.3\ttrain-error:0.107778\n",
      "[9]\teval-error:0.31\ttrain-error:0.101111\n",
      "[10]\teval-error:0.31\ttrain-error:0.093333\n",
      "[11]\teval-error:0.31\ttrain-error:0.092222\n",
      "[12]\teval-error:0.3\ttrain-error:0.087778\n",
      "[13]\teval-error:0.29\ttrain-error:0.081111\n",
      "[14]\teval-error:0.29\ttrain-error:0.076667\n",
      "[15]\teval-error:0.28\ttrain-error:0.077778\n",
      "[16]\teval-error:0.29\ttrain-error:0.073333\n",
      "[17]\teval-error:0.29\ttrain-error:0.073333\n",
      "[18]\teval-error:0.28\ttrain-error:0.072222\n",
      "[19]\teval-error:0.26\ttrain-error:0.068889\n",
      "[20]\teval-error:0.26\ttrain-error:0.067778\n",
      "[21]\teval-error:0.27\ttrain-error:0.067778\n",
      "[22]\teval-error:0.27\ttrain-error:0.065556\n",
      "[23]\teval-error:0.26\ttrain-error:0.062222\n",
      "[24]\teval-error:0.26\ttrain-error:0.06\n",
      "[25]\teval-error:0.27\ttrain-error:0.06\n",
      "[26]\teval-error:0.25\ttrain-error:0.056667\n",
      "[27]\teval-error:0.25\ttrain-error:0.052222\n",
      "[28]\teval-error:0.26\ttrain-error:0.048889\n",
      "[29]\teval-error:0.26\ttrain-error:0.046667\n",
      "[30]\teval-error:0.25\ttrain-error:0.043333\n",
      "[31]\teval-error:0.25\ttrain-error:0.042222\n",
      "[32]\teval-error:0.25\ttrain-error:0.04\n",
      "[33]\teval-error:0.25\ttrain-error:0.04\n",
      "[34]\teval-error:0.26\ttrain-error:0.035556\n",
      "[35]\teval-error:0.26\ttrain-error:0.035556\n",
      "[36]\teval-error:0.26\ttrain-error:0.031111\n",
      "[37]\teval-error:0.25\ttrain-error:0.031111\n",
      "[38]\teval-error:0.25\ttrain-error:0.03\n",
      "[39]\teval-error:0.26\ttrain-error:0.028889\n",
      "[40]\teval-error:0.26\ttrain-error:0.027778\n",
      "[41]\teval-error:0.26\ttrain-error:0.027778\n",
      "[42]\teval-error:0.25\ttrain-error:0.026667\n",
      "[43]\teval-error:0.25\ttrain-error:0.025556\n",
      "[44]\teval-error:0.25\ttrain-error:0.026667\n",
      "[45]\teval-error:0.25\ttrain-error:0.024444\n",
      "[46]\teval-error:0.25\ttrain-error:0.023333\n",
      "[47]\teval-error:0.26\ttrain-error:0.024444\n",
      "[48]\teval-error:0.25\ttrain-error:0.023333\n",
      "[49]\teval-error:0.25\ttrain-error:0.023333\n",
      "[50]\teval-error:0.25\ttrain-error:0.022222\n",
      "[51]\teval-error:0.25\ttrain-error:0.021111\n",
      "[52]\teval-error:0.25\ttrain-error:0.02\n",
      "[53]\teval-error:0.25\ttrain-error:0.02\n",
      "[54]\teval-error:0.24\ttrain-error:0.018889\n",
      "[55]\teval-error:0.24\ttrain-error:0.015556\n",
      "[56]\teval-error:0.24\ttrain-error:0.015556\n",
      "[57]\teval-error:0.24\ttrain-error:0.014444\n",
      "[58]\teval-error:0.25\ttrain-error:0.015556\n",
      "[59]\teval-error:0.24\ttrain-error:0.015556\n",
      "[60]\teval-error:0.24\ttrain-error:0.015556\n",
      "[61]\teval-error:0.25\ttrain-error:0.012222\n",
      "[62]\teval-error:0.25\ttrain-error:0.011111\n",
      "[63]\teval-error:0.25\ttrain-error:0.011111\n",
      "[64]\teval-error:0.25\ttrain-error:0.011111\n",
      "[65]\teval-error:0.25\ttrain-error:0.01\n",
      "[66]\teval-error:0.25\ttrain-error:0.008889\n",
      "[67]\teval-error:0.25\ttrain-error:0.008889\n",
      "[68]\teval-error:0.25\ttrain-error:0.007778\n",
      "[69]\teval-error:0.25\ttrain-error:0.007778\n",
      "[70]\teval-error:0.25\ttrain-error:0.006667\n",
      "[71]\teval-error:0.25\ttrain-error:0.006667\n",
      "[72]\teval-error:0.25\ttrain-error:0.006667\n",
      "[73]\teval-error:0.25\ttrain-error:0.006667\n",
      "[74]\teval-error:0.25\ttrain-error:0.006667\n",
      "[75]\teval-error:0.25\ttrain-error:0.006667\n",
      "[76]\teval-error:0.25\ttrain-error:0.006667\n",
      "[77]\teval-error:0.24\ttrain-error:0.006667\n",
      "[78]\teval-error:0.25\ttrain-error:0.005556\n",
      "[79]\teval-error:0.24\ttrain-error:0.005556\n",
      "[80]\teval-error:0.24\ttrain-error:0.005556\n",
      "[81]\teval-error:0.24\ttrain-error:0.005556\n",
      "[82]\teval-error:0.24\ttrain-error:0.004444\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[83]\teval-error:0.24\ttrain-error:0.004444\n",
      "[84]\teval-error:0.24\ttrain-error:0.004444\n",
      "[85]\teval-error:0.24\ttrain-error:0.004444\n",
      "[86]\teval-error:0.24\ttrain-error:0.004444\n",
      "[87]\teval-error:0.24\ttrain-error:0.004444\n",
      "[88]\teval-error:0.24\ttrain-error:0.004444\n",
      "[89]\teval-error:0.24\ttrain-error:0.004444\n",
      "[90]\teval-error:0.24\ttrain-error:0.003333\n",
      "[91]\teval-error:0.24\ttrain-error:0.003333\n",
      "[92]\teval-error:0.24\ttrain-error:0.003333\n",
      "[93]\teval-error:0.24\ttrain-error:0.003333\n",
      "[94]\teval-error:0.24\ttrain-error:0.003333\n",
      "[95]\teval-error:0.24\ttrain-error:0.003333\n",
      "[96]\teval-error:0.24\ttrain-error:0.003333\n",
      "[97]\teval-error:0.24\ttrain-error:0.003333\n",
      "[98]\teval-error:0.24\ttrain-error:0.002222\n",
      "[99]\teval-error:0.24\ttrain-error:0.002222\n",
      "[100]\teval-error:0.24\ttrain-error:0.002222\n",
      "[101]\teval-error:0.24\ttrain-error:0.002222\n",
      "[102]\teval-error:0.24\ttrain-error:0.002222\n",
      "[103]\teval-error:0.24\ttrain-error:0.002222\n",
      "[104]\teval-error:0.24\ttrain-error:0.002222\n",
      "[105]\teval-error:0.24\ttrain-error:0.002222\n",
      "[106]\teval-error:0.24\ttrain-error:0.002222\n",
      "[107]\teval-error:0.24\ttrain-error:0.002222\n",
      "[108]\teval-error:0.24\ttrain-error:0.002222\n",
      "Stopping. Best iteration:\n",
      "[98]\teval-error:0.24\ttrain-error:0.002222\n",
      "\n",
      "[0 1 0 0 0 0 1 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0\n",
      " 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 0 1 0 1 0 0 0 0 0 0 0\n",
      " 0 0 1 0 0 0 0 0 0 1 0 1 0 0 1 1 1 0 0 0 0 0 0 0 1 0]\n",
      "[0]\teval-error:0.31\ttrain-error:0.15\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.32\ttrain-error:0.125556\n",
      "[2]\teval-error:0.31\ttrain-error:0.116667\n",
      "[3]\teval-error:0.3\ttrain-error:0.116667\n",
      "[4]\teval-error:0.3\ttrain-error:0.111111\n",
      "[5]\teval-error:0.29\ttrain-error:0.107778\n",
      "[6]\teval-error:0.3\ttrain-error:0.108889\n",
      "[7]\teval-error:0.31\ttrain-error:0.095556\n",
      "[8]\teval-error:0.29\ttrain-error:0.092222\n",
      "[9]\teval-error:0.29\ttrain-error:0.094444\n",
      "[10]\teval-error:0.28\ttrain-error:0.085556\n",
      "[11]\teval-error:0.31\ttrain-error:0.088889\n",
      "[12]\teval-error:0.29\ttrain-error:0.084444\n",
      "[13]\teval-error:0.29\ttrain-error:0.085556\n",
      "[14]\teval-error:0.26\ttrain-error:0.076667\n",
      "[15]\teval-error:0.28\ttrain-error:0.071111\n",
      "[16]\teval-error:0.28\ttrain-error:0.064444\n",
      "[17]\teval-error:0.28\ttrain-error:0.068889\n",
      "[18]\teval-error:0.27\ttrain-error:0.064444\n",
      "[19]\teval-error:0.26\ttrain-error:0.061111\n",
      "[20]\teval-error:0.27\ttrain-error:0.061111\n",
      "[21]\teval-error:0.27\ttrain-error:0.061111\n",
      "[22]\teval-error:0.27\ttrain-error:0.056667\n",
      "[23]\teval-error:0.27\ttrain-error:0.055556\n",
      "[24]\teval-error:0.27\ttrain-error:0.053333\n",
      "[25]\teval-error:0.26\ttrain-error:0.05\n",
      "[26]\teval-error:0.26\ttrain-error:0.048889\n",
      "[27]\teval-error:0.28\ttrain-error:0.047778\n",
      "[28]\teval-error:0.28\ttrain-error:0.047778\n",
      "[29]\teval-error:0.27\ttrain-error:0.047778\n",
      "[30]\teval-error:0.28\ttrain-error:0.05\n",
      "[31]\teval-error:0.29\ttrain-error:0.05\n",
      "[32]\teval-error:0.29\ttrain-error:0.05\n",
      "[33]\teval-error:0.29\ttrain-error:0.048889\n",
      "[34]\teval-error:0.29\ttrain-error:0.048889\n",
      "[35]\teval-error:0.29\ttrain-error:0.048889\n",
      "[36]\teval-error:0.28\ttrain-error:0.044444\n",
      "[37]\teval-error:0.29\ttrain-error:0.043333\n",
      "[38]\teval-error:0.29\ttrain-error:0.044444\n",
      "[39]\teval-error:0.29\ttrain-error:0.043333\n",
      "[40]\teval-error:0.29\ttrain-error:0.04\n",
      "[41]\teval-error:0.29\ttrain-error:0.035556\n",
      "[42]\teval-error:0.28\ttrain-error:0.033333\n",
      "[43]\teval-error:0.27\ttrain-error:0.028889\n",
      "[44]\teval-error:0.27\ttrain-error:0.027778\n",
      "[45]\teval-error:0.27\ttrain-error:0.024444\n",
      "[46]\teval-error:0.27\ttrain-error:0.023333\n",
      "[47]\teval-error:0.27\ttrain-error:0.025556\n",
      "[48]\teval-error:0.27\ttrain-error:0.025556\n",
      "[49]\teval-error:0.27\ttrain-error:0.024444\n",
      "[50]\teval-error:0.27\ttrain-error:0.022222\n",
      "[51]\teval-error:0.27\ttrain-error:0.02\n",
      "[52]\teval-error:0.28\ttrain-error:0.018889\n",
      "[53]\teval-error:0.28\ttrain-error:0.016667\n",
      "[54]\teval-error:0.28\ttrain-error:0.016667\n",
      "[55]\teval-error:0.28\ttrain-error:0.018889\n",
      "[56]\teval-error:0.27\ttrain-error:0.017778\n",
      "[57]\teval-error:0.28\ttrain-error:0.014444\n",
      "[58]\teval-error:0.28\ttrain-error:0.012222\n",
      "[59]\teval-error:0.26\ttrain-error:0.01\n",
      "[60]\teval-error:0.27\ttrain-error:0.01\n",
      "[61]\teval-error:0.26\ttrain-error:0.007778\n",
      "[62]\teval-error:0.26\ttrain-error:0.006667\n",
      "[63]\teval-error:0.26\ttrain-error:0.004444\n",
      "[64]\teval-error:0.26\ttrain-error:0.004444\n",
      "[65]\teval-error:0.27\ttrain-error:0.004444\n",
      "[66]\teval-error:0.28\ttrain-error:0.004444\n",
      "[67]\teval-error:0.28\ttrain-error:0.004444\n",
      "[68]\teval-error:0.28\ttrain-error:0.004444\n",
      "[69]\teval-error:0.28\ttrain-error:0.003333\n",
      "[70]\teval-error:0.28\ttrain-error:0.004444\n",
      "[71]\teval-error:0.28\ttrain-error:0.003333\n",
      "[72]\teval-error:0.28\ttrain-error:0.003333\n",
      "[73]\teval-error:0.28\ttrain-error:0.003333\n",
      "[74]\teval-error:0.28\ttrain-error:0.003333\n",
      "[75]\teval-error:0.28\ttrain-error:0.003333\n",
      "[76]\teval-error:0.27\ttrain-error:0.002222\n",
      "[77]\teval-error:0.27\ttrain-error:0.002222\n",
      "[78]\teval-error:0.28\ttrain-error:0.002222\n",
      "[79]\teval-error:0.27\ttrain-error:0.002222\n",
      "[80]\teval-error:0.27\ttrain-error:0.002222\n",
      "[81]\teval-error:0.27\ttrain-error:0.002222\n",
      "[82]\teval-error:0.27\ttrain-error:0.002222\n",
      "[83]\teval-error:0.27\ttrain-error:0.001111\n",
      "[84]\teval-error:0.28\ttrain-error:0.001111\n",
      "[85]\teval-error:0.29\ttrain-error:0.001111\n",
      "[86]\teval-error:0.29\ttrain-error:0.001111\n",
      "[87]\teval-error:0.29\ttrain-error:0.001111\n",
      "[88]\teval-error:0.29\ttrain-error:0.001111\n",
      "[89]\teval-error:0.29\ttrain-error:0.001111\n",
      "[90]\teval-error:0.29\ttrain-error:0.001111\n",
      "[91]\teval-error:0.29\ttrain-error:0.001111\n",
      "[92]\teval-error:0.3\ttrain-error:0.001111\n",
      "[93]\teval-error:0.29\ttrain-error:0.001111\n",
      "Stopping. Best iteration:\n",
      "[83]\teval-error:0.27\ttrain-error:0.001111\n",
      "\n",
      "[0 1 0 0 1 1 0 0 0 0 1 0 1 0 0 0 1 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 0 0 1\n",
      " 0 0 0 1 0 0 1 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 1 0\n",
      " 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 1 0 0]\n",
      "[0]\teval-error:0.26\ttrain-error:0.158889\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.27\ttrain-error:0.146667\n",
      "[2]\teval-error:0.26\ttrain-error:0.127778\n",
      "[3]\teval-error:0.31\ttrain-error:0.132222\n",
      "[4]\teval-error:0.33\ttrain-error:0.112222\n",
      "[5]\teval-error:0.31\ttrain-error:0.117778\n",
      "[6]\teval-error:0.3\ttrain-error:0.112222\n",
      "[7]\teval-error:0.28\ttrain-error:0.103333\n",
      "[8]\teval-error:0.25\ttrain-error:0.103333\n",
      "[9]\teval-error:0.26\ttrain-error:0.1\n",
      "[10]\teval-error:0.29\ttrain-error:0.096667\n",
      "[11]\teval-error:0.3\ttrain-error:0.092222\n",
      "[12]\teval-error:0.3\ttrain-error:0.085556\n",
      "[13]\teval-error:0.3\ttrain-error:0.083333\n",
      "[14]\teval-error:0.3\ttrain-error:0.082222\n",
      "[15]\teval-error:0.29\ttrain-error:0.08\n",
      "[16]\teval-error:0.28\ttrain-error:0.076667\n",
      "[17]\teval-error:0.27\ttrain-error:0.078889\n",
      "[18]\teval-error:0.28\ttrain-error:0.075556\n",
      "[19]\teval-error:0.29\ttrain-error:0.07\n",
      "[20]\teval-error:0.28\ttrain-error:0.07\n",
      "[21]\teval-error:0.28\ttrain-error:0.064444\n",
      "[22]\teval-error:0.27\ttrain-error:0.064444\n",
      "[23]\teval-error:0.28\ttrain-error:0.062222\n",
      "[24]\teval-error:0.27\ttrain-error:0.062222\n",
      "[25]\teval-error:0.27\ttrain-error:0.06\n",
      "[26]\teval-error:0.28\ttrain-error:0.057778\n",
      "[27]\teval-error:0.26\ttrain-error:0.055556\n",
      "[28]\teval-error:0.26\ttrain-error:0.055556\n",
      "[29]\teval-error:0.27\ttrain-error:0.053333\n",
      "[30]\teval-error:0.27\ttrain-error:0.048889\n",
      "[31]\teval-error:0.27\ttrain-error:0.048889\n",
      "[32]\teval-error:0.27\ttrain-error:0.045556\n",
      "[33]\teval-error:0.27\ttrain-error:0.046667\n",
      "[34]\teval-error:0.27\ttrain-error:0.043333\n",
      "[35]\teval-error:0.27\ttrain-error:0.04\n",
      "[36]\teval-error:0.28\ttrain-error:0.04\n",
      "[37]\teval-error:0.28\ttrain-error:0.037778\n",
      "[38]\teval-error:0.27\ttrain-error:0.038889\n",
      "[39]\teval-error:0.27\ttrain-error:0.034444\n",
      "[40]\teval-error:0.27\ttrain-error:0.033333\n",
      "[41]\teval-error:0.27\ttrain-error:0.032222\n",
      "[42]\teval-error:0.27\ttrain-error:0.032222\n",
      "[43]\teval-error:0.27\ttrain-error:0.028889\n",
      "[44]\teval-error:0.28\ttrain-error:0.03\n",
      "[45]\teval-error:0.28\ttrain-error:0.026667\n",
      "[46]\teval-error:0.28\ttrain-error:0.024444\n",
      "[47]\teval-error:0.28\ttrain-error:0.025556\n",
      "[48]\teval-error:0.28\ttrain-error:0.021111\n",
      "[49]\teval-error:0.27\ttrain-error:0.018889\n",
      "[50]\teval-error:0.29\ttrain-error:0.02\n",
      "[51]\teval-error:0.29\ttrain-error:0.018889\n",
      "[52]\teval-error:0.29\ttrain-error:0.017778\n",
      "[53]\teval-error:0.28\ttrain-error:0.018889\n",
      "[54]\teval-error:0.29\ttrain-error:0.016667\n",
      "[55]\teval-error:0.28\ttrain-error:0.016667\n",
      "[56]\teval-error:0.27\ttrain-error:0.015556\n",
      "[57]\teval-error:0.26\ttrain-error:0.014444\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[58]\teval-error:0.27\ttrain-error:0.012222\n",
      "[59]\teval-error:0.25\ttrain-error:0.012222\n",
      "[60]\teval-error:0.26\ttrain-error:0.011111\n",
      "[61]\teval-error:0.26\ttrain-error:0.011111\n",
      "[62]\teval-error:0.25\ttrain-error:0.011111\n",
      "[63]\teval-error:0.26\ttrain-error:0.011111\n",
      "[64]\teval-error:0.26\ttrain-error:0.008889\n",
      "[65]\teval-error:0.26\ttrain-error:0.008889\n",
      "[66]\teval-error:0.26\ttrain-error:0.008889\n",
      "[67]\teval-error:0.26\ttrain-error:0.007778\n",
      "[68]\teval-error:0.26\ttrain-error:0.006667\n",
      "[69]\teval-error:0.26\ttrain-error:0.006667\n",
      "[70]\teval-error:0.26\ttrain-error:0.006667\n",
      "[71]\teval-error:0.26\ttrain-error:0.006667\n",
      "[72]\teval-error:0.25\ttrain-error:0.006667\n",
      "[73]\teval-error:0.25\ttrain-error:0.004444\n",
      "[74]\teval-error:0.24\ttrain-error:0.004444\n",
      "[75]\teval-error:0.24\ttrain-error:0.004444\n",
      "[76]\teval-error:0.24\ttrain-error:0.004444\n",
      "[77]\teval-error:0.24\ttrain-error:0.004444\n",
      "[78]\teval-error:0.24\ttrain-error:0.004444\n",
      "[79]\teval-error:0.24\ttrain-error:0.004444\n",
      "[80]\teval-error:0.24\ttrain-error:0.004444\n",
      "[81]\teval-error:0.24\ttrain-error:0.004444\n",
      "[82]\teval-error:0.24\ttrain-error:0.003333\n",
      "[83]\teval-error:0.24\ttrain-error:0.003333\n",
      "[84]\teval-error:0.24\ttrain-error:0.003333\n",
      "[85]\teval-error:0.24\ttrain-error:0.003333\n",
      "[86]\teval-error:0.24\ttrain-error:0.003333\n",
      "[87]\teval-error:0.24\ttrain-error:0.003333\n",
      "[88]\teval-error:0.24\ttrain-error:0.003333\n",
      "[89]\teval-error:0.24\ttrain-error:0.003333\n",
      "[90]\teval-error:0.24\ttrain-error:0.003333\n",
      "[91]\teval-error:0.24\ttrain-error:0.003333\n",
      "[92]\teval-error:0.25\ttrain-error:0.003333\n",
      "Stopping. Best iteration:\n",
      "[82]\teval-error:0.24\ttrain-error:0.003333\n",
      "\n",
      "[0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 1 0 1 1 0 0 0 0 0 1 0 0 0 0 1 0 1 0 1\n",
      " 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 1 0 1 1 1 0 0\n",
      " 0 0 1 0 0 1 1 0 0 0 0 1 0 0 1 0 1 0 0 0 1 0 0 0 0 0]\n",
      "[0]\teval-error:0.34\ttrain-error:0.14\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.32\ttrain-error:0.121111\n",
      "[2]\teval-error:0.31\ttrain-error:0.115556\n",
      "[3]\teval-error:0.29\ttrain-error:0.108889\n",
      "[4]\teval-error:0.32\ttrain-error:0.104444\n",
      "[5]\teval-error:0.32\ttrain-error:0.095556\n",
      "[6]\teval-error:0.31\ttrain-error:0.094444\n",
      "[7]\teval-error:0.34\ttrain-error:0.087778\n",
      "[8]\teval-error:0.34\ttrain-error:0.081111\n",
      "[9]\teval-error:0.33\ttrain-error:0.085556\n",
      "[10]\teval-error:0.33\ttrain-error:0.08\n",
      "[11]\teval-error:0.33\ttrain-error:0.08\n",
      "[12]\teval-error:0.33\ttrain-error:0.081111\n",
      "[13]\teval-error:0.33\ttrain-error:0.076667\n",
      "[14]\teval-error:0.33\ttrain-error:0.071111\n",
      "[15]\teval-error:0.33\ttrain-error:0.071111\n",
      "[16]\teval-error:0.33\ttrain-error:0.067778\n",
      "[17]\teval-error:0.33\ttrain-error:0.066667\n",
      "[18]\teval-error:0.33\ttrain-error:0.064444\n",
      "[19]\teval-error:0.32\ttrain-error:0.06\n",
      "[20]\teval-error:0.33\ttrain-error:0.057778\n",
      "[21]\teval-error:0.33\ttrain-error:0.058889\n",
      "[22]\teval-error:0.33\ttrain-error:0.057778\n",
      "[23]\teval-error:0.33\ttrain-error:0.055556\n",
      "[24]\teval-error:0.33\ttrain-error:0.057778\n",
      "[25]\teval-error:0.33\ttrain-error:0.055556\n",
      "[26]\teval-error:0.33\ttrain-error:0.056667\n",
      "[27]\teval-error:0.33\ttrain-error:0.054444\n",
      "[28]\teval-error:0.33\ttrain-error:0.054444\n",
      "[29]\teval-error:0.32\ttrain-error:0.05\n",
      "[30]\teval-error:0.32\ttrain-error:0.045556\n",
      "[31]\teval-error:0.32\ttrain-error:0.044444\n",
      "[32]\teval-error:0.32\ttrain-error:0.041111\n",
      "[33]\teval-error:0.32\ttrain-error:0.037778\n",
      "[34]\teval-error:0.32\ttrain-error:0.035556\n",
      "[35]\teval-error:0.32\ttrain-error:0.034444\n",
      "[36]\teval-error:0.32\ttrain-error:0.034444\n",
      "[37]\teval-error:0.32\ttrain-error:0.033333\n",
      "[38]\teval-error:0.33\ttrain-error:0.034444\n",
      "[39]\teval-error:0.31\ttrain-error:0.031111\n",
      "[40]\teval-error:0.31\ttrain-error:0.031111\n",
      "[41]\teval-error:0.3\ttrain-error:0.028889\n",
      "[42]\teval-error:0.32\ttrain-error:0.026667\n",
      "[43]\teval-error:0.33\ttrain-error:0.024444\n",
      "[44]\teval-error:0.31\ttrain-error:0.024444\n",
      "[45]\teval-error:0.34\ttrain-error:0.023333\n",
      "[46]\teval-error:0.33\ttrain-error:0.025556\n",
      "[47]\teval-error:0.33\ttrain-error:0.022222\n",
      "[48]\teval-error:0.33\ttrain-error:0.02\n",
      "[49]\teval-error:0.33\ttrain-error:0.02\n",
      "[50]\teval-error:0.33\ttrain-error:0.02\n",
      "[51]\teval-error:0.33\ttrain-error:0.02\n",
      "[52]\teval-error:0.33\ttrain-error:0.018889\n",
      "[53]\teval-error:0.33\ttrain-error:0.018889\n",
      "[54]\teval-error:0.33\ttrain-error:0.017778\n",
      "[55]\teval-error:0.32\ttrain-error:0.016667\n",
      "[56]\teval-error:0.33\ttrain-error:0.015556\n",
      "[57]\teval-error:0.33\ttrain-error:0.015556\n",
      "[58]\teval-error:0.33\ttrain-error:0.015556\n",
      "[59]\teval-error:0.33\ttrain-error:0.013333\n",
      "[60]\teval-error:0.33\ttrain-error:0.013333\n",
      "[61]\teval-error:0.32\ttrain-error:0.013333\n",
      "[62]\teval-error:0.33\ttrain-error:0.013333\n",
      "[63]\teval-error:0.33\ttrain-error:0.011111\n",
      "[64]\teval-error:0.33\ttrain-error:0.011111\n",
      "[65]\teval-error:0.31\ttrain-error:0.01\n",
      "[66]\teval-error:0.32\ttrain-error:0.008889\n",
      "[67]\teval-error:0.32\ttrain-error:0.008889\n",
      "[68]\teval-error:0.32\ttrain-error:0.007778\n",
      "[69]\teval-error:0.31\ttrain-error:0.007778\n",
      "[70]\teval-error:0.31\ttrain-error:0.007778\n",
      "[71]\teval-error:0.31\ttrain-error:0.007778\n",
      "[72]\teval-error:0.31\ttrain-error:0.006667\n",
      "[73]\teval-error:0.3\ttrain-error:0.006667\n",
      "[74]\teval-error:0.29\ttrain-error:0.007778\n",
      "[75]\teval-error:0.29\ttrain-error:0.006667\n",
      "[76]\teval-error:0.29\ttrain-error:0.006667\n",
      "[77]\teval-error:0.3\ttrain-error:0.006667\n",
      "[78]\teval-error:0.3\ttrain-error:0.005556\n",
      "[79]\teval-error:0.3\ttrain-error:0.003333\n",
      "[80]\teval-error:0.3\ttrain-error:0.005556\n",
      "[81]\teval-error:0.31\ttrain-error:0.003333\n",
      "[82]\teval-error:0.3\ttrain-error:0.003333\n",
      "[83]\teval-error:0.3\ttrain-error:0.003333\n",
      "[84]\teval-error:0.31\ttrain-error:0.003333\n",
      "[85]\teval-error:0.31\ttrain-error:0.004444\n",
      "[86]\teval-error:0.29\ttrain-error:0.003333\n",
      "[87]\teval-error:0.3\ttrain-error:0.003333\n",
      "[88]\teval-error:0.29\ttrain-error:0.003333\n",
      "[89]\teval-error:0.29\ttrain-error:0.003333\n",
      "Stopping. Best iteration:\n",
      "[79]\teval-error:0.3\ttrain-error:0.003333\n",
      "\n",
      "[1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 1 0 1 0 0 1 0 0 0 0 1 0\n",
      " 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 1 0 0 0 0 0 0 1 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 1 0 1 1 0 0 1 0]\n",
      "[0]\teval-error:0.32\ttrain-error:0.147778\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.31\ttrain-error:0.136667\n",
      "[2]\teval-error:0.36\ttrain-error:0.124444\n",
      "[3]\teval-error:0.35\ttrain-error:0.114444\n",
      "[4]\teval-error:0.35\ttrain-error:0.113333\n",
      "[5]\teval-error:0.38\ttrain-error:0.102222\n",
      "[6]\teval-error:0.35\ttrain-error:0.101111\n",
      "[7]\teval-error:0.34\ttrain-error:0.1\n",
      "[8]\teval-error:0.33\ttrain-error:0.1\n",
      "[9]\teval-error:0.34\ttrain-error:0.092222\n",
      "[10]\teval-error:0.34\ttrain-error:0.091111\n",
      "[11]\teval-error:0.32\ttrain-error:0.081111\n",
      "[12]\teval-error:0.36\ttrain-error:0.081111\n",
      "[13]\teval-error:0.36\ttrain-error:0.076667\n",
      "[14]\teval-error:0.37\ttrain-error:0.077778\n",
      "[15]\teval-error:0.36\ttrain-error:0.074444\n",
      "[16]\teval-error:0.35\ttrain-error:0.072222\n",
      "[17]\teval-error:0.36\ttrain-error:0.063333\n",
      "[18]\teval-error:0.34\ttrain-error:0.063333\n",
      "[19]\teval-error:0.34\ttrain-error:0.063333\n",
      "[20]\teval-error:0.33\ttrain-error:0.062222\n",
      "[21]\teval-error:0.33\ttrain-error:0.061111\n",
      "[22]\teval-error:0.32\ttrain-error:0.06\n",
      "[23]\teval-error:0.32\ttrain-error:0.058889\n",
      "[24]\teval-error:0.34\ttrain-error:0.058889\n",
      "[25]\teval-error:0.33\ttrain-error:0.06\n",
      "[26]\teval-error:0.32\ttrain-error:0.058889\n",
      "[27]\teval-error:0.32\ttrain-error:0.058889\n",
      "[28]\teval-error:0.32\ttrain-error:0.056667\n",
      "[29]\teval-error:0.32\ttrain-error:0.052222\n",
      "[30]\teval-error:0.34\ttrain-error:0.05\n",
      "[31]\teval-error:0.34\ttrain-error:0.048889\n",
      "[32]\teval-error:0.34\ttrain-error:0.048889\n",
      "[33]\teval-error:0.35\ttrain-error:0.042222\n",
      "[34]\teval-error:0.35\ttrain-error:0.042222\n",
      "[35]\teval-error:0.36\ttrain-error:0.038889\n",
      "[36]\teval-error:0.36\ttrain-error:0.036667\n",
      "[37]\teval-error:0.36\ttrain-error:0.037778\n",
      "[38]\teval-error:0.36\ttrain-error:0.036667\n",
      "[39]\teval-error:0.35\ttrain-error:0.034444\n",
      "[40]\teval-error:0.36\ttrain-error:0.031111\n",
      "[41]\teval-error:0.36\ttrain-error:0.03\n",
      "[42]\teval-error:0.36\ttrain-error:0.03\n",
      "[43]\teval-error:0.36\ttrain-error:0.028889\n",
      "[44]\teval-error:0.36\ttrain-error:0.026667\n",
      "[45]\teval-error:0.37\ttrain-error:0.026667\n",
      "[46]\teval-error:0.37\ttrain-error:0.024444\n",
      "[47]\teval-error:0.38\ttrain-error:0.024444\n",
      "[48]\teval-error:0.38\ttrain-error:0.022222\n",
      "[49]\teval-error:0.38\ttrain-error:0.021111\n",
      "[50]\teval-error:0.39\ttrain-error:0.018889\n",
      "[51]\teval-error:0.4\ttrain-error:0.018889\n",
      "[52]\teval-error:0.39\ttrain-error:0.018889\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[53]\teval-error:0.39\ttrain-error:0.017778\n",
      "[54]\teval-error:0.39\ttrain-error:0.018889\n",
      "[55]\teval-error:0.39\ttrain-error:0.018889\n",
      "[56]\teval-error:0.39\ttrain-error:0.017778\n",
      "[57]\teval-error:0.39\ttrain-error:0.016667\n",
      "[58]\teval-error:0.4\ttrain-error:0.015556\n",
      "[59]\teval-error:0.39\ttrain-error:0.015556\n",
      "[60]\teval-error:0.39\ttrain-error:0.014444\n",
      "[61]\teval-error:0.38\ttrain-error:0.014444\n",
      "[62]\teval-error:0.38\ttrain-error:0.014444\n",
      "[63]\teval-error:0.38\ttrain-error:0.014444\n",
      "[64]\teval-error:0.38\ttrain-error:0.014444\n",
      "[65]\teval-error:0.39\ttrain-error:0.011111\n",
      "[66]\teval-error:0.39\ttrain-error:0.011111\n",
      "[67]\teval-error:0.39\ttrain-error:0.01\n",
      "[68]\teval-error:0.39\ttrain-error:0.01\n",
      "[69]\teval-error:0.39\ttrain-error:0.008889\n",
      "[70]\teval-error:0.39\ttrain-error:0.006667\n",
      "[71]\teval-error:0.39\ttrain-error:0.007778\n",
      "[72]\teval-error:0.39\ttrain-error:0.006667\n",
      "[73]\teval-error:0.39\ttrain-error:0.006667\n",
      "[74]\teval-error:0.39\ttrain-error:0.006667\n",
      "[75]\teval-error:0.39\ttrain-error:0.006667\n",
      "[76]\teval-error:0.39\ttrain-error:0.005556\n",
      "[77]\teval-error:0.39\ttrain-error:0.005556\n",
      "[78]\teval-error:0.39\ttrain-error:0.004444\n",
      "[79]\teval-error:0.39\ttrain-error:0.004444\n",
      "[80]\teval-error:0.39\ttrain-error:0.004444\n",
      "[81]\teval-error:0.39\ttrain-error:0.004444\n",
      "[82]\teval-error:0.39\ttrain-error:0.004444\n",
      "[83]\teval-error:0.39\ttrain-error:0.004444\n",
      "[84]\teval-error:0.39\ttrain-error:0.003333\n",
      "[85]\teval-error:0.39\ttrain-error:0.003333\n",
      "[86]\teval-error:0.39\ttrain-error:0.002222\n",
      "[87]\teval-error:0.39\ttrain-error:0.002222\n",
      "[88]\teval-error:0.39\ttrain-error:0.002222\n",
      "[89]\teval-error:0.39\ttrain-error:0.002222\n",
      "[90]\teval-error:0.39\ttrain-error:0.002222\n",
      "[91]\teval-error:0.39\ttrain-error:0.002222\n",
      "[92]\teval-error:0.39\ttrain-error:0.002222\n",
      "[93]\teval-error:0.39\ttrain-error:0.002222\n",
      "[94]\teval-error:0.39\ttrain-error:0.001111\n",
      "[95]\teval-error:0.39\ttrain-error:0.001111\n",
      "[96]\teval-error:0.39\ttrain-error:0.001111\n",
      "[97]\teval-error:0.39\ttrain-error:0.001111\n",
      "[98]\teval-error:0.39\ttrain-error:0.001111\n",
      "[99]\teval-error:0.39\ttrain-error:0.001111\n",
      "[100]\teval-error:0.39\ttrain-error:0.001111\n",
      "[101]\teval-error:0.39\ttrain-error:0\n",
      "[102]\teval-error:0.39\ttrain-error:0\n",
      "[103]\teval-error:0.39\ttrain-error:0\n",
      "[104]\teval-error:0.39\ttrain-error:0\n",
      "[105]\teval-error:0.39\ttrain-error:0\n",
      "[106]\teval-error:0.42\ttrain-error:0\n",
      "[107]\teval-error:0.42\ttrain-error:0\n",
      "[108]\teval-error:0.39\ttrain-error:0\n",
      "[109]\teval-error:0.41\ttrain-error:0\n",
      "[110]\teval-error:0.41\ttrain-error:0\n",
      "[111]\teval-error:0.41\ttrain-error:0\n",
      "Stopping. Best iteration:\n",
      "[101]\teval-error:0.39\ttrain-error:0\n",
      "\n",
      "[1 1 0 1 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 1 1 0 0 0 0 0 0 0 1 0 0 1 0 1 0 0\n",
      " 0 0 0 0 0 0 1 1 0 0 1 0 0 1 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 1\n",
      " 0 0 0 0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1]\n",
      "[0]\teval-error:0.15\ttrain-error:0.154444\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.16\ttrain-error:0.146667\n",
      "[2]\teval-error:0.16\ttrain-error:0.124444\n",
      "[3]\teval-error:0.19\ttrain-error:0.124444\n",
      "[4]\teval-error:0.18\ttrain-error:0.121111\n",
      "[5]\teval-error:0.18\ttrain-error:0.12\n",
      "[6]\teval-error:0.18\ttrain-error:0.112222\n",
      "[7]\teval-error:0.2\ttrain-error:0.112222\n",
      "[8]\teval-error:0.18\ttrain-error:0.113333\n",
      "[9]\teval-error:0.18\ttrain-error:0.107778\n",
      "[10]\teval-error:0.17\ttrain-error:0.094444\n",
      "[11]\teval-error:0.19\ttrain-error:0.093333\n",
      "[12]\teval-error:0.18\ttrain-error:0.09\n",
      "[13]\teval-error:0.15\ttrain-error:0.086667\n",
      "[14]\teval-error:0.17\ttrain-error:0.085556\n",
      "[15]\teval-error:0.18\ttrain-error:0.081111\n",
      "[16]\teval-error:0.18\ttrain-error:0.083333\n",
      "[17]\teval-error:0.17\ttrain-error:0.073333\n",
      "[18]\teval-error:0.17\ttrain-error:0.068889\n",
      "[19]\teval-error:0.17\ttrain-error:0.063333\n",
      "[20]\teval-error:0.17\ttrain-error:0.063333\n",
      "[21]\teval-error:0.19\ttrain-error:0.064444\n",
      "[22]\teval-error:0.18\ttrain-error:0.064444\n",
      "[23]\teval-error:0.17\ttrain-error:0.063333\n",
      "[24]\teval-error:0.19\ttrain-error:0.062222\n",
      "[25]\teval-error:0.17\ttrain-error:0.058889\n",
      "[26]\teval-error:0.19\ttrain-error:0.057778\n",
      "[27]\teval-error:0.17\ttrain-error:0.058889\n",
      "[28]\teval-error:0.17\ttrain-error:0.053333\n",
      "[29]\teval-error:0.16\ttrain-error:0.053333\n",
      "[30]\teval-error:0.17\ttrain-error:0.053333\n",
      "[31]\teval-error:0.17\ttrain-error:0.053333\n",
      "[32]\teval-error:0.17\ttrain-error:0.051111\n",
      "[33]\teval-error:0.17\ttrain-error:0.045556\n",
      "[34]\teval-error:0.17\ttrain-error:0.044444\n",
      "[35]\teval-error:0.16\ttrain-error:0.044444\n",
      "[36]\teval-error:0.16\ttrain-error:0.041111\n",
      "[37]\teval-error:0.17\ttrain-error:0.04\n",
      "[38]\teval-error:0.18\ttrain-error:0.034444\n",
      "[39]\teval-error:0.17\ttrain-error:0.033333\n",
      "[40]\teval-error:0.16\ttrain-error:0.032222\n",
      "[41]\teval-error:0.16\ttrain-error:0.033333\n",
      "[42]\teval-error:0.17\ttrain-error:0.033333\n",
      "[43]\teval-error:0.17\ttrain-error:0.028889\n",
      "[44]\teval-error:0.16\ttrain-error:0.027778\n",
      "[45]\teval-error:0.16\ttrain-error:0.026667\n",
      "[46]\teval-error:0.16\ttrain-error:0.025556\n",
      "[47]\teval-error:0.16\ttrain-error:0.025556\n",
      "[48]\teval-error:0.15\ttrain-error:0.023333\n",
      "[49]\teval-error:0.15\ttrain-error:0.021111\n",
      "[50]\teval-error:0.14\ttrain-error:0.021111\n",
      "[51]\teval-error:0.15\ttrain-error:0.02\n",
      "[52]\teval-error:0.14\ttrain-error:0.018889\n",
      "[53]\teval-error:0.14\ttrain-error:0.017778\n",
      "[54]\teval-error:0.14\ttrain-error:0.016667\n",
      "[55]\teval-error:0.14\ttrain-error:0.015556\n",
      "[56]\teval-error:0.14\ttrain-error:0.015556\n",
      "[57]\teval-error:0.14\ttrain-error:0.015556\n",
      "[58]\teval-error:0.14\ttrain-error:0.015556\n",
      "[59]\teval-error:0.14\ttrain-error:0.011111\n",
      "[60]\teval-error:0.14\ttrain-error:0.011111\n",
      "[61]\teval-error:0.14\ttrain-error:0.011111\n",
      "[62]\teval-error:0.15\ttrain-error:0.01\n",
      "[63]\teval-error:0.15\ttrain-error:0.008889\n",
      "[64]\teval-error:0.15\ttrain-error:0.006667\n",
      "[65]\teval-error:0.15\ttrain-error:0.005556\n",
      "[66]\teval-error:0.15\ttrain-error:0.005556\n",
      "[67]\teval-error:0.15\ttrain-error:0.005556\n",
      "[68]\teval-error:0.15\ttrain-error:0.005556\n",
      "[69]\teval-error:0.14\ttrain-error:0.005556\n",
      "[70]\teval-error:0.15\ttrain-error:0.005556\n",
      "[71]\teval-error:0.14\ttrain-error:0.005556\n",
      "[72]\teval-error:0.14\ttrain-error:0.005556\n",
      "[73]\teval-error:0.14\ttrain-error:0.005556\n",
      "[74]\teval-error:0.14\ttrain-error:0.004444\n",
      "[75]\teval-error:0.14\ttrain-error:0.003333\n",
      "[76]\teval-error:0.14\ttrain-error:0.003333\n",
      "[77]\teval-error:0.14\ttrain-error:0.003333\n",
      "[78]\teval-error:0.14\ttrain-error:0.003333\n",
      "[79]\teval-error:0.14\ttrain-error:0.003333\n",
      "[80]\teval-error:0.14\ttrain-error:0.003333\n",
      "[81]\teval-error:0.14\ttrain-error:0.003333\n",
      "[82]\teval-error:0.14\ttrain-error:0.003333\n",
      "[83]\teval-error:0.15\ttrain-error:0.003333\n",
      "[84]\teval-error:0.14\ttrain-error:0.003333\n",
      "[85]\teval-error:0.14\ttrain-error:0.003333\n",
      "Stopping. Best iteration:\n",
      "[75]\teval-error:0.14\ttrain-error:0.003333\n",
      "\n",
      "[0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 1 0 0 1 1 0 0 0 0 0 0 0 0 0 1\n",
      " 0 1 1 0 0 0 0 0 1 0 0 1 0 0 0 0 1 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 1 1 0 1 0\n",
      " 0 0 0 0 1 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 1 0 1]\n",
      "[0]\teval-error:0.37\ttrain-error:0.144444\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.32\ttrain-error:0.131111\n",
      "[2]\teval-error:0.31\ttrain-error:0.121111\n",
      "[3]\teval-error:0.29\ttrain-error:0.122222\n",
      "[4]\teval-error:0.29\ttrain-error:0.12\n",
      "[5]\teval-error:0.3\ttrain-error:0.121111\n",
      "[6]\teval-error:0.27\ttrain-error:0.113333\n",
      "[7]\teval-error:0.29\ttrain-error:0.108889\n",
      "[8]\teval-error:0.28\ttrain-error:0.103333\n",
      "[9]\teval-error:0.27\ttrain-error:0.1\n",
      "[10]\teval-error:0.27\ttrain-error:0.096667\n",
      "[11]\teval-error:0.29\ttrain-error:0.098889\n",
      "[12]\teval-error:0.26\ttrain-error:0.1\n",
      "[13]\teval-error:0.28\ttrain-error:0.093333\n",
      "[14]\teval-error:0.27\ttrain-error:0.086667\n",
      "[15]\teval-error:0.27\ttrain-error:0.084444\n",
      "[16]\teval-error:0.26\ttrain-error:0.081111\n",
      "[17]\teval-error:0.26\ttrain-error:0.075556\n",
      "[18]\teval-error:0.26\ttrain-error:0.071111\n",
      "[19]\teval-error:0.26\ttrain-error:0.07\n",
      "[20]\teval-error:0.26\ttrain-error:0.07\n",
      "[21]\teval-error:0.26\ttrain-error:0.07\n",
      "[22]\teval-error:0.26\ttrain-error:0.067778\n",
      "[23]\teval-error:0.26\ttrain-error:0.066667\n",
      "[24]\teval-error:0.26\ttrain-error:0.066667\n",
      "[25]\teval-error:0.25\ttrain-error:0.063333\n",
      "[26]\teval-error:0.27\ttrain-error:0.061111\n",
      "[27]\teval-error:0.26\ttrain-error:0.058889\n",
      "[28]\teval-error:0.27\ttrain-error:0.057778\n",
      "[29]\teval-error:0.25\ttrain-error:0.051111\n",
      "[30]\teval-error:0.26\ttrain-error:0.052222\n",
      "[31]\teval-error:0.27\ttrain-error:0.048889\n",
      "[32]\teval-error:0.27\ttrain-error:0.048889\n",
      "[33]\teval-error:0.26\ttrain-error:0.048889\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[34]\teval-error:0.26\ttrain-error:0.048889\n",
      "[35]\teval-error:0.26\ttrain-error:0.048889\n",
      "[36]\teval-error:0.27\ttrain-error:0.048889\n",
      "[37]\teval-error:0.26\ttrain-error:0.045556\n",
      "[38]\teval-error:0.25\ttrain-error:0.043333\n",
      "[39]\teval-error:0.25\ttrain-error:0.043333\n",
      "[40]\teval-error:0.26\ttrain-error:0.04\n",
      "[41]\teval-error:0.26\ttrain-error:0.037778\n",
      "[42]\teval-error:0.26\ttrain-error:0.036667\n",
      "[43]\teval-error:0.26\ttrain-error:0.035556\n",
      "[44]\teval-error:0.26\ttrain-error:0.035556\n",
      "[45]\teval-error:0.26\ttrain-error:0.035556\n",
      "[46]\teval-error:0.26\ttrain-error:0.036667\n",
      "[47]\teval-error:0.26\ttrain-error:0.037778\n",
      "[48]\teval-error:0.26\ttrain-error:0.035556\n",
      "[49]\teval-error:0.26\ttrain-error:0.034444\n",
      "[50]\teval-error:0.26\ttrain-error:0.033333\n",
      "[51]\teval-error:0.26\ttrain-error:0.032222\n",
      "[52]\teval-error:0.26\ttrain-error:0.031111\n",
      "[53]\teval-error:0.26\ttrain-error:0.028889\n",
      "[54]\teval-error:0.26\ttrain-error:0.027778\n",
      "[55]\teval-error:0.26\ttrain-error:0.026667\n",
      "[56]\teval-error:0.26\ttrain-error:0.025556\n",
      "[57]\teval-error:0.26\ttrain-error:0.024444\n",
      "[58]\teval-error:0.26\ttrain-error:0.024444\n",
      "[59]\teval-error:0.26\ttrain-error:0.023333\n",
      "[60]\teval-error:0.26\ttrain-error:0.022222\n",
      "[61]\teval-error:0.26\ttrain-error:0.022222\n",
      "[62]\teval-error:0.26\ttrain-error:0.022222\n",
      "[63]\teval-error:0.27\ttrain-error:0.021111\n",
      "[64]\teval-error:0.27\ttrain-error:0.02\n",
      "[65]\teval-error:0.27\ttrain-error:0.02\n",
      "[66]\teval-error:0.28\ttrain-error:0.018889\n",
      "[67]\teval-error:0.28\ttrain-error:0.017778\n",
      "[68]\teval-error:0.28\ttrain-error:0.018889\n",
      "[69]\teval-error:0.27\ttrain-error:0.015556\n",
      "[70]\teval-error:0.27\ttrain-error:0.012222\n",
      "[71]\teval-error:0.28\ttrain-error:0.011111\n",
      "[72]\teval-error:0.28\ttrain-error:0.012222\n",
      "[73]\teval-error:0.28\ttrain-error:0.01\n",
      "[74]\teval-error:0.28\ttrain-error:0.01\n",
      "[75]\teval-error:0.28\ttrain-error:0.008889\n",
      "[76]\teval-error:0.28\ttrain-error:0.008889\n",
      "[77]\teval-error:0.29\ttrain-error:0.008889\n",
      "[78]\teval-error:0.29\ttrain-error:0.007778\n",
      "[79]\teval-error:0.29\ttrain-error:0.007778\n",
      "[80]\teval-error:0.29\ttrain-error:0.007778\n",
      "[81]\teval-error:0.29\ttrain-error:0.007778\n",
      "[82]\teval-error:0.29\ttrain-error:0.007778\n",
      "[83]\teval-error:0.29\ttrain-error:0.006667\n",
      "[84]\teval-error:0.29\ttrain-error:0.007778\n",
      "[85]\teval-error:0.29\ttrain-error:0.006667\n",
      "[86]\teval-error:0.29\ttrain-error:0.006667\n",
      "[87]\teval-error:0.29\ttrain-error:0.005556\n",
      "[88]\teval-error:0.29\ttrain-error:0.004444\n",
      "[89]\teval-error:0.29\ttrain-error:0.003333\n",
      "[90]\teval-error:0.29\ttrain-error:0.002222\n",
      "[91]\teval-error:0.29\ttrain-error:0.002222\n",
      "[92]\teval-error:0.29\ttrain-error:0.002222\n",
      "[93]\teval-error:0.29\ttrain-error:0.002222\n",
      "[94]\teval-error:0.29\ttrain-error:0.002222\n",
      "[95]\teval-error:0.29\ttrain-error:0.002222\n",
      "[96]\teval-error:0.29\ttrain-error:0.002222\n",
      "[97]\teval-error:0.29\ttrain-error:0.002222\n",
      "[98]\teval-error:0.29\ttrain-error:0.002222\n",
      "[99]\teval-error:0.3\ttrain-error:0.002222\n",
      "[100]\teval-error:0.29\ttrain-error:0.002222\n",
      "Stopping. Best iteration:\n",
      "[90]\teval-error:0.29\ttrain-error:0.002222\n",
      "\n",
      "[0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 1\n",
      " 0 0 1 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 0 0 0 1 0 0 0 1 1 1 0 0 1 1 0 1 0 0\n",
      " 1 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 1 1 1 0 0]\n",
      "[0]\teval-error:0.27\ttrain-error:0.182222\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.29\ttrain-error:0.16\n",
      "[2]\teval-error:0.29\ttrain-error:0.143333\n",
      "[3]\teval-error:0.29\ttrain-error:0.14\n",
      "[4]\teval-error:0.29\ttrain-error:0.136667\n",
      "[5]\teval-error:0.27\ttrain-error:0.121111\n",
      "[6]\teval-error:0.26\ttrain-error:0.118889\n",
      "[7]\teval-error:0.26\ttrain-error:0.11\n",
      "[8]\teval-error:0.25\ttrain-error:0.11\n",
      "[9]\teval-error:0.24\ttrain-error:0.098889\n",
      "[10]\teval-error:0.23\ttrain-error:0.091111\n",
      "[11]\teval-error:0.25\ttrain-error:0.09\n",
      "[12]\teval-error:0.23\ttrain-error:0.084444\n",
      "[13]\teval-error:0.23\ttrain-error:0.085556\n",
      "[14]\teval-error:0.24\ttrain-error:0.082222\n",
      "[15]\teval-error:0.24\ttrain-error:0.084444\n",
      "[16]\teval-error:0.24\ttrain-error:0.082222\n",
      "[17]\teval-error:0.25\ttrain-error:0.074444\n",
      "[18]\teval-error:0.25\ttrain-error:0.073333\n",
      "[19]\teval-error:0.25\ttrain-error:0.071111\n",
      "[20]\teval-error:0.25\ttrain-error:0.066667\n",
      "[21]\teval-error:0.25\ttrain-error:0.064444\n",
      "[22]\teval-error:0.25\ttrain-error:0.064444\n",
      "[23]\teval-error:0.26\ttrain-error:0.066667\n",
      "[24]\teval-error:0.26\ttrain-error:0.066667\n",
      "[25]\teval-error:0.26\ttrain-error:0.064444\n",
      "[26]\teval-error:0.26\ttrain-error:0.063333\n",
      "[27]\teval-error:0.26\ttrain-error:0.061111\n",
      "[28]\teval-error:0.26\ttrain-error:0.056667\n",
      "[29]\teval-error:0.26\ttrain-error:0.055556\n",
      "[30]\teval-error:0.26\ttrain-error:0.05\n",
      "[31]\teval-error:0.26\ttrain-error:0.048889\n",
      "[32]\teval-error:0.26\ttrain-error:0.048889\n",
      "[33]\teval-error:0.26\ttrain-error:0.044444\n",
      "[34]\teval-error:0.26\ttrain-error:0.044444\n",
      "[35]\teval-error:0.27\ttrain-error:0.038889\n",
      "[36]\teval-error:0.27\ttrain-error:0.038889\n",
      "[37]\teval-error:0.27\ttrain-error:0.037778\n",
      "[38]\teval-error:0.27\ttrain-error:0.037778\n",
      "[39]\teval-error:0.27\ttrain-error:0.037778\n",
      "[40]\teval-error:0.27\ttrain-error:0.035556\n",
      "[41]\teval-error:0.27\ttrain-error:0.034444\n",
      "[42]\teval-error:0.27\ttrain-error:0.033333\n",
      "[43]\teval-error:0.27\ttrain-error:0.028889\n",
      "[44]\teval-error:0.27\ttrain-error:0.026667\n",
      "[45]\teval-error:0.26\ttrain-error:0.025556\n",
      "[46]\teval-error:0.27\ttrain-error:0.024444\n",
      "[47]\teval-error:0.27\ttrain-error:0.022222\n",
      "[48]\teval-error:0.27\ttrain-error:0.022222\n",
      "[49]\teval-error:0.27\ttrain-error:0.018889\n",
      "[50]\teval-error:0.27\ttrain-error:0.017778\n",
      "[51]\teval-error:0.27\ttrain-error:0.017778\n",
      "[52]\teval-error:0.27\ttrain-error:0.017778\n",
      "[53]\teval-error:0.27\ttrain-error:0.016667\n",
      "[54]\teval-error:0.27\ttrain-error:0.016667\n",
      "[55]\teval-error:0.27\ttrain-error:0.016667\n",
      "[56]\teval-error:0.27\ttrain-error:0.016667\n",
      "[57]\teval-error:0.28\ttrain-error:0.016667\n",
      "[58]\teval-error:0.28\ttrain-error:0.016667\n",
      "[59]\teval-error:0.28\ttrain-error:0.016667\n",
      "[60]\teval-error:0.28\ttrain-error:0.014444\n",
      "[61]\teval-error:0.28\ttrain-error:0.014444\n",
      "[62]\teval-error:0.28\ttrain-error:0.012222\n",
      "[63]\teval-error:0.27\ttrain-error:0.013333\n",
      "[64]\teval-error:0.28\ttrain-error:0.013333\n",
      "[65]\teval-error:0.28\ttrain-error:0.01\n",
      "[66]\teval-error:0.28\ttrain-error:0.01\n",
      "[67]\teval-error:0.28\ttrain-error:0.01\n",
      "[68]\teval-error:0.29\ttrain-error:0.006667\n",
      "[69]\teval-error:0.28\ttrain-error:0.006667\n",
      "[70]\teval-error:0.28\ttrain-error:0.006667\n",
      "[71]\teval-error:0.28\ttrain-error:0.006667\n",
      "[72]\teval-error:0.28\ttrain-error:0.006667\n",
      "[73]\teval-error:0.29\ttrain-error:0.006667\n",
      "[74]\teval-error:0.29\ttrain-error:0.005556\n",
      "[75]\teval-error:0.29\ttrain-error:0.005556\n",
      "[76]\teval-error:0.29\ttrain-error:0.005556\n",
      "[77]\teval-error:0.28\ttrain-error:0.005556\n",
      "[78]\teval-error:0.28\ttrain-error:0.004444\n",
      "[79]\teval-error:0.28\ttrain-error:0.004444\n",
      "[80]\teval-error:0.28\ttrain-error:0.004444\n",
      "[81]\teval-error:0.28\ttrain-error:0.004444\n",
      "[82]\teval-error:0.28\ttrain-error:0.004444\n",
      "[83]\teval-error:0.28\ttrain-error:0.004444\n",
      "[84]\teval-error:0.29\ttrain-error:0.004444\n",
      "[85]\teval-error:0.29\ttrain-error:0.004444\n",
      "[86]\teval-error:0.29\ttrain-error:0.004444\n",
      "[87]\teval-error:0.29\ttrain-error:0.003333\n",
      "[88]\teval-error:0.29\ttrain-error:0.003333\n",
      "[89]\teval-error:0.29\ttrain-error:0.003333\n",
      "[90]\teval-error:0.29\ttrain-error:0.003333\n",
      "[91]\teval-error:0.28\ttrain-error:0.003333\n",
      "[92]\teval-error:0.28\ttrain-error:0.003333\n",
      "[93]\teval-error:0.28\ttrain-error:0.003333\n",
      "[94]\teval-error:0.28\ttrain-error:0.003333\n",
      "[95]\teval-error:0.28\ttrain-error:0.003333\n",
      "[96]\teval-error:0.28\ttrain-error:0.003333\n",
      "[97]\teval-error:0.28\ttrain-error:0.002222\n",
      "[98]\teval-error:0.28\ttrain-error:0.002222\n",
      "[99]\teval-error:0.29\ttrain-error:0.001111\n",
      "[100]\teval-error:0.29\ttrain-error:0.001111\n",
      "[101]\teval-error:0.29\ttrain-error:0.001111\n",
      "[102]\teval-error:0.29\ttrain-error:0.001111\n",
      "[103]\teval-error:0.28\ttrain-error:0.001111\n",
      "[104]\teval-error:0.28\ttrain-error:0.001111\n",
      "[105]\teval-error:0.29\ttrain-error:0.001111\n",
      "[106]\teval-error:0.29\ttrain-error:0.001111\n",
      "[107]\teval-error:0.28\ttrain-error:0.001111\n",
      "[108]\teval-error:0.28\ttrain-error:0.001111\n",
      "[109]\teval-error:0.28\ttrain-error:0.001111\n",
      "Stopping. Best iteration:\n",
      "[99]\teval-error:0.29\ttrain-error:0.001111\n",
      "\n",
      "[1 1 0 0 1 0 1 0 1 0 0 1 1 0 1 0 0 0 1 1 0 0 1 0 0 1 1 0 0 1 0 0 0 0 1 1 0\n",
      " 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0\n",
      " 0 0 1 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0]\n",
      "[0]\teval-error:0.29\ttrain-error:0.148889\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will train until train-error hasn't improved in 10 rounds.\n",
      "[1]\teval-error:0.25\ttrain-error:0.137778\n",
      "[2]\teval-error:0.24\ttrain-error:0.131111\n",
      "[3]\teval-error:0.24\ttrain-error:0.125556\n",
      "[4]\teval-error:0.26\ttrain-error:0.125556\n",
      "[5]\teval-error:0.29\ttrain-error:0.11\n",
      "[6]\teval-error:0.24\ttrain-error:0.111111\n",
      "[7]\teval-error:0.29\ttrain-error:0.101111\n",
      "[8]\teval-error:0.29\ttrain-error:0.092222\n",
      "[9]\teval-error:0.29\ttrain-error:0.088889\n",
      "[10]\teval-error:0.28\ttrain-error:0.083333\n",
      "[11]\teval-error:0.28\ttrain-error:0.076667\n",
      "[12]\teval-error:0.27\ttrain-error:0.083333\n",
      "[13]\teval-error:0.26\ttrain-error:0.08\n",
      "[14]\teval-error:0.28\ttrain-error:0.076667\n",
      "[15]\teval-error:0.29\ttrain-error:0.076667\n",
      "[16]\teval-error:0.29\ttrain-error:0.076667\n",
      "[17]\teval-error:0.29\ttrain-error:0.073333\n",
      "[18]\teval-error:0.29\ttrain-error:0.067778\n",
      "[19]\teval-error:0.28\ttrain-error:0.066667\n",
      "[20]\teval-error:0.29\ttrain-error:0.063333\n",
      "[21]\teval-error:0.29\ttrain-error:0.065556\n",
      "[22]\teval-error:0.29\ttrain-error:0.067778\n",
      "[23]\teval-error:0.29\ttrain-error:0.066667\n",
      "[24]\teval-error:0.3\ttrain-error:0.061111\n",
      "[25]\teval-error:0.29\ttrain-error:0.06\n",
      "[26]\teval-error:0.28\ttrain-error:0.06\n",
      "[27]\teval-error:0.28\ttrain-error:0.055556\n",
      "[28]\teval-error:0.28\ttrain-error:0.054444\n",
      "[29]\teval-error:0.29\ttrain-error:0.05\n",
      "[30]\teval-error:0.28\ttrain-error:0.046667\n",
      "[31]\teval-error:0.29\ttrain-error:0.047778\n",
      "[32]\teval-error:0.28\ttrain-error:0.044444\n",
      "[33]\teval-error:0.27\ttrain-error:0.044444\n",
      "[34]\teval-error:0.26\ttrain-error:0.044444\n",
      "[35]\teval-error:0.26\ttrain-error:0.044444\n",
      "[36]\teval-error:0.26\ttrain-error:0.041111\n",
      "[37]\teval-error:0.26\ttrain-error:0.04\n",
      "[38]\teval-error:0.26\ttrain-error:0.04\n",
      "[39]\teval-error:0.26\ttrain-error:0.036667\n",
      "[40]\teval-error:0.26\ttrain-error:0.036667\n",
      "[41]\teval-error:0.26\ttrain-error:0.035556\n",
      "[42]\teval-error:0.26\ttrain-error:0.032222\n",
      "[43]\teval-error:0.27\ttrain-error:0.032222\n",
      "[44]\teval-error:0.26\ttrain-error:0.031111\n",
      "[45]\teval-error:0.27\ttrain-error:0.031111\n",
      "[46]\teval-error:0.27\ttrain-error:0.031111\n",
      "[47]\teval-error:0.27\ttrain-error:0.031111\n",
      "[48]\teval-error:0.25\ttrain-error:0.031111\n",
      "[49]\teval-error:0.26\ttrain-error:0.031111\n",
      "[50]\teval-error:0.26\ttrain-error:0.03\n",
      "[51]\teval-error:0.27\ttrain-error:0.027778\n",
      "[52]\teval-error:0.26\ttrain-error:0.025556\n",
      "[53]\teval-error:0.26\ttrain-error:0.023333\n",
      "[54]\teval-error:0.25\ttrain-error:0.023333\n",
      "[55]\teval-error:0.25\ttrain-error:0.024444\n",
      "[56]\teval-error:0.25\ttrain-error:0.024444\n",
      "[57]\teval-error:0.25\ttrain-error:0.022222\n",
      "[58]\teval-error:0.25\ttrain-error:0.021111\n",
      "[59]\teval-error:0.26\ttrain-error:0.018889\n",
      "[60]\teval-error:0.26\ttrain-error:0.016667\n",
      "[61]\teval-error:0.26\ttrain-error:0.016667\n",
      "[62]\teval-error:0.26\ttrain-error:0.016667\n",
      "[63]\teval-error:0.26\ttrain-error:0.015556\n",
      "[64]\teval-error:0.25\ttrain-error:0.014444\n",
      "[65]\teval-error:0.26\ttrain-error:0.014444\n",
      "[66]\teval-error:0.26\ttrain-error:0.012222\n",
      "[67]\teval-error:0.26\ttrain-error:0.012222\n",
      "[68]\teval-error:0.26\ttrain-error:0.012222\n",
      "[69]\teval-error:0.26\ttrain-error:0.012222\n",
      "[70]\teval-error:0.25\ttrain-error:0.011111\n",
      "[71]\teval-error:0.26\ttrain-error:0.01\n",
      "[72]\teval-error:0.26\ttrain-error:0.007778\n",
      "[73]\teval-error:0.25\ttrain-error:0.007778\n",
      "[74]\teval-error:0.25\ttrain-error:0.007778\n",
      "[75]\teval-error:0.26\ttrain-error:0.005556\n",
      "[76]\teval-error:0.25\ttrain-error:0.005556\n",
      "[77]\teval-error:0.25\ttrain-error:0.005556\n",
      "[78]\teval-error:0.24\ttrain-error:0.005556\n",
      "[79]\teval-error:0.24\ttrain-error:0.005556\n",
      "[80]\teval-error:0.24\ttrain-error:0.005556\n",
      "[81]\teval-error:0.24\ttrain-error:0.005556\n",
      "[82]\teval-error:0.25\ttrain-error:0.005556\n",
      "[83]\teval-error:0.26\ttrain-error:0.005556\n",
      "[84]\teval-error:0.25\ttrain-error:0.005556\n",
      "[85]\teval-error:0.24\ttrain-error:0.004444\n",
      "[86]\teval-error:0.23\ttrain-error:0.004444\n",
      "[87]\teval-error:0.24\ttrain-error:0.004444\n",
      "[88]\teval-error:0.23\ttrain-error:0.004444\n",
      "[89]\teval-error:0.24\ttrain-error:0.004444\n",
      "[90]\teval-error:0.24\ttrain-error:0.004444\n",
      "[91]\teval-error:0.24\ttrain-error:0.004444\n",
      "[92]\teval-error:0.24\ttrain-error:0.004444\n",
      "[93]\teval-error:0.24\ttrain-error:0.004444\n",
      "[94]\teval-error:0.24\ttrain-error:0.004444\n",
      "[95]\teval-error:0.23\ttrain-error:0.004444\n",
      "Stopping. Best iteration:\n",
      "[85]\teval-error:0.24\ttrain-error:0.004444\n",
      "\n",
      "[1 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 0 0 1 0 0 0 0 0 1 1 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 1 0 0 1 0 0 1 0 0 0 1 1 0 0 0 1\n",
      " 0 1 0 1 0 1 1 0 0 1 0 1 0 1 1 0 0 0 0 0 0 0 0 1 0 1]\n"
     ]
    }
   ],
   "source": [
    "for train_index, test_index in kf.split(X):\n",
    "    data_train = xgb.DMatrix(X[train_index], y[train_index])\n",
    "    data_test = xgb.DMatrix(X[test_index],y[test_index])\n",
    "    watch_list = [(data_test,\"eval\"),(data_train,\"train\")]\n",
    "    bst = xgb.train(params,data_train,num_boost_round=20000,evals = watch_list,early_stopping_rounds = 10)\n",
    "    ypred = bst.predict(data_test)\n",
    "    y_pred = (ypred >= 0.5)*1\n",
    "    print(y_pred)\n",
    "    auc_score.append(metrics.roc_auc_score(y[test_index],ypred))\n",
    "    acc_score.append(metrics.accuracy_score(y[test_index],y_pred))\n",
    "    recall_score.append(metrics.recall_score(y[test_index],y_pred))\n",
    "    prec_score.append(metrics.precision_score(y[test_index],y_pred))\n",
    "    f1_score.append(metrics.f1_score(y[test_index],y_pred))\n",
    "    beta_score.append(fbeta_score(y[test_index], y_pred, beta=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR: 0.467397 (0.063640)\n",
      "RF: 0.347039 (0.105530)\n",
      "SVM: 0.266798 (0.114255)\n"
     ]
    }
   ],
   "source": [
    "# to feed the random state\n",
    "seed = 7\n",
    "# prepare models\n",
    "models = []\n",
    "models.append(('LR', LogisticRegression()))\n",
    "#models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "#models.append(('KNN', KNeighborsClassifier()))\n",
    "#models.append(('CART', DecisionTreeClassifier()))\n",
    "#models.append(('NB', GaussianNB()))\n",
    "models.append(('RF', RandomForestClassifier()))\n",
    "models.append(('SVM', SVC(gamma='auto')))\n",
    "#models.append(('XGB', XGBClassifier()))\n",
    "\n",
    "# evaluate each model in turn\n",
    "results = []\n",
    "names = []\n",
    "scoring = 'f1'\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state=42)\n",
    "for name, model in models:\n",
    "        kfold = KFold(n_splits=10, random_state=42)\n",
    "        cv_results = cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)\n",
    "        results.append(cv_results)\n",
    "        names.append(name)\n",
    "        msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "        print(msg)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compare_result = pd.DataFrame(results).T\n",
    "compare_result.columns= names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compare_result[\"XGBOOST\"] = pd.DataFrame(f1_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LR</th>\n",
       "      <th>RF</th>\n",
       "      <th>SVM</th>\n",
       "      <th>XGBOOST</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.176471</td>\n",
       "      <td>0.294118</td>\n",
       "      <td>0.576923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.580645</td>\n",
       "      <td>0.275862</td>\n",
       "      <td>0.173913</td>\n",
       "      <td>0.538462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.242424</td>\n",
       "      <td>0.491228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.378378</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.509804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.372093</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.431373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.457143</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.296296</td>\n",
       "      <td>0.349206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.432432</td>\n",
       "      <td>0.451613</td>\n",
       "      <td>0.148148</td>\n",
       "      <td>0.708333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.451613</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.472727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.594595</td>\n",
       "      <td>0.424242</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.410256</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.634921</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         LR        RF       SVM   XGBOOST\n",
       "0  0.450000  0.176471  0.294118  0.576923\n",
       "1  0.580645  0.275862  0.173913  0.538462\n",
       "2  0.461538  0.263158  0.242424  0.491228\n",
       "3  0.444444  0.378378  0.285714  0.509804\n",
       "4  0.391304  0.372093  0.263158  0.431373\n",
       "5  0.457143  0.200000  0.296296  0.349206\n",
       "6  0.432432  0.451613  0.148148  0.708333\n",
       "7  0.451613  0.500000  0.428571  0.472727\n",
       "8  0.594595  0.424242  0.466667  0.461538\n",
       "9  0.410256  0.428571  0.068966  0.634921"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "names.append(\"xgboost\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "marker": {
          "color": "#3D9970"
         },
         "name": "LR",
         "type": "box",
         "y": [
          0.45,
          0.5806451612903226,
          0.46153846153846156,
          0.4444444444444444,
          0.391304347826087,
          0.45714285714285713,
          0.43243243243243246,
          0.45161290322580644,
          0.5945945945945946,
          0.41025641025641024
         ]
        },
        {
         "marker": {
          "color": "#3D9900"
         },
         "name": "RF",
         "type": "box",
         "x": "SVM",
         "y": [
          0.17647058823529413,
          0.27586206896551724,
          0.26315789473684204,
          0.3783783783783784,
          0.37209302325581395,
          0.19999999999999998,
          0.45161290322580644,
          0.5,
          0.4242424242424242,
          0.42857142857142855
         ]
        },
        {
         "marker": {
          "color": "#FF4136"
         },
         "name": "SVM",
         "type": "box",
         "x": "SVM",
         "y": [
          0.2941176470588235,
          0.17391304347826086,
          0.24242424242424243,
          0.28571428571428575,
          0.26315789473684215,
          0.2962962962962963,
          0.14814814814814814,
          0.4285714285714285,
          0.4666666666666666,
          0.06896551724137931
         ]
        },
        {
         "marker": {
          "color": "#3D0070"
         },
         "name": "XGBOOST",
         "type": "box",
         "x": "SVM",
         "y": [
          0.576923076923077,
          0.5384615384615385,
          0.4912280701754386,
          0.5098039215686274,
          0.4313725490196078,
          0.34920634920634924,
          0.7083333333333334,
          0.4727272727272727,
          0.4615384615384615,
          0.634920634920635
         ]
        }
       ],
       "layout": {
        "boxmode": "group",
        "xaxis": {
         "title": "Age Categorical"
        },
        "yaxis": {
         "title": "Credit Amount (US Dollar)",
         "zeroline": false
        }
       }
      },
      "text/html": [
       "<div id=\"12ea1756-f8f2-464f-8c66-889124612bcd\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"12ea1756-f8f2-464f-8c66-889124612bcd\", [{\"type\": \"box\", \"y\": [0.45, 0.5806451612903226, 0.46153846153846156, 0.4444444444444444, 0.391304347826087, 0.45714285714285713, 0.43243243243243246, 0.45161290322580644, 0.5945945945945946, 0.41025641025641024], \"name\": \"LR\", \"marker\": {\"color\": \"#3D9970\"}}, {\"type\": \"box\", \"y\": [0.17647058823529413, 0.27586206896551724, 0.26315789473684204, 0.3783783783783784, 0.37209302325581395, 0.19999999999999998, 0.45161290322580644, 0.5, 0.4242424242424242, 0.42857142857142855], \"x\": \"SVM\", \"name\": \"RF\", \"marker\": {\"color\": \"#3D9900\"}}, {\"type\": \"box\", \"y\": [0.2941176470588235, 0.17391304347826086, 0.24242424242424243, 0.28571428571428575, 0.26315789473684215, 0.2962962962962963, 0.14814814814814814, 0.4285714285714285, 0.4666666666666666, 0.06896551724137931], \"x\": \"SVM\", \"name\": \"SVM\", \"marker\": {\"color\": \"#FF4136\"}}, {\"type\": \"box\", \"y\": [0.576923076923077, 0.5384615384615385, 0.4912280701754386, 0.5098039215686274, 0.4313725490196078, 0.34920634920634924, 0.7083333333333334, 0.4727272727272727, 0.4615384615384615, 0.634920634920635], \"x\": \"SVM\", \"name\": \"XGBOOST\", \"marker\": {\"color\": \"#3D0070\"}}], {\"yaxis\": {\"title\": \"Credit Amount (US Dollar)\", \"zeroline\": false}, \"xaxis\": {\"title\": \"Age Categorical\"}, \"boxmode\": \"group\"}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"12ea1756-f8f2-464f-8c66-889124612bcd\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"12ea1756-f8f2-464f-8c66-889124612bcd\", [{\"type\": \"box\", \"y\": [0.45, 0.5806451612903226, 0.46153846153846156, 0.4444444444444444, 0.391304347826087, 0.45714285714285713, 0.43243243243243246, 0.45161290322580644, 0.5945945945945946, 0.41025641025641024], \"name\": \"LR\", \"marker\": {\"color\": \"#3D9970\"}}, {\"type\": \"box\", \"y\": [0.17647058823529413, 0.27586206896551724, 0.26315789473684204, 0.3783783783783784, 0.37209302325581395, 0.19999999999999998, 0.45161290322580644, 0.5, 0.4242424242424242, 0.42857142857142855], \"x\": \"SVM\", \"name\": \"RF\", \"marker\": {\"color\": \"#3D9900\"}}, {\"type\": \"box\", \"y\": [0.2941176470588235, 0.17391304347826086, 0.24242424242424243, 0.28571428571428575, 0.26315789473684215, 0.2962962962962963, 0.14814814814814814, 0.4285714285714285, 0.4666666666666666, 0.06896551724137931], \"x\": \"SVM\", \"name\": \"SVM\", \"marker\": {\"color\": \"#FF4136\"}}, {\"type\": \"box\", \"y\": [0.576923076923077, 0.5384615384615385, 0.4912280701754386, 0.5098039215686274, 0.4313725490196078, 0.34920634920634924, 0.7083333333333334, 0.4727272727272727, 0.4615384615384615, 0.634920634920635], \"x\": \"SVM\", \"name\": \"XGBOOST\", \"marker\": {\"color\": \"#3D0070\"}}], {\"yaxis\": {\"title\": \"Credit Amount (US Dollar)\", \"zeroline\": false}, \"xaxis\": {\"title\": \"Age Categorical\"}, \"boxmode\": \"group\"}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# it's a library that we work with plotly\n",
    "import plotly.offline as py \n",
    "py.init_notebook_mode(connected=True) # this code, allow us to work with offline plotly version\n",
    "import plotly.graph_objs as go # it's like \"plt\" of matplot\n",
    "import plotly.tools as tls # It's useful to we get some tools of plotly\n",
    "import warnings # This library will be used to ignore some warnings\n",
    "from collections import Counter # To do counter of some features\n",
    "\n",
    "trace0 = go.Box(\n",
    "    y= compare_result[\"LR\"].values.tolist(),\n",
    "    name='LR',\n",
    "    marker=dict(\n",
    "        color='#3D9970'\n",
    "    )\n",
    ")\n",
    "\n",
    "trace1 = go.Box(\n",
    "    y= compare_result[\"RF\"].values.tolist(),\n",
    "    x= name,\n",
    "    name='RF',\n",
    "    marker=dict(\n",
    "        color='#3D9900'\n",
    "    )\n",
    ")\n",
    "\n",
    "trace2 = go.Box(\n",
    "    y= compare_result[\"SVM\"].values.tolist(),\n",
    "    x= name,\n",
    "    name='SVM',\n",
    "    marker=dict(\n",
    "        color='#FF4136'\n",
    "    )\n",
    ")\n",
    "trace3 = go.Box(\n",
    "    y= compare_result[\"XGBOOST\"].values.tolist(),\n",
    "    x= name,\n",
    "    name='XGBOOST',\n",
    "    marker=dict(\n",
    "        color='#3D0070'\n",
    "    )\n",
    ")\n",
    "   \n",
    "data = [trace0,trace1,trace2,trace3]\n",
    "\n",
    "layout = go.Layout(\n",
    "    yaxis=dict(\n",
    "        title='Credit Amount (US Dollar)',\n",
    "        zeroline=False\n",
    "    ),\n",
    "    xaxis=dict(\n",
    "        title='Age Categorical'\n",
    "    ),\n",
    "    boxmode='group'\n",
    ")\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "\n",
    "py.iplot(fig, filename='box-age-cat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([ 0.45      ,  0.58064516,  0.46153846,  0.44444444,  0.39130435,\n",
       "         0.45714286,  0.43243243,  0.4516129 ,  0.59459459,  0.41025641]),\n",
       " array([ 0.42857143,  0.58064516,  0.5       ,  0.46153846,  0.38297872,\n",
       "         0.48648649,  0.43902439,  0.45714286,  0.66666667,  0.47619048]),\n",
       " array([ 0.35      ,  0.32258065,  0.23809524,  0.35294118,  0.31818182,\n",
       "         0.25806452,  0.25641026,  0.36363636,  0.375     ,  0.40909091]),\n",
       " array([ 0.625     ,  0.57777778,  0.35087719,  0.41860465,  0.48979592,\n",
       "         0.55813953,  0.43478261,  0.40816327,  0.49056604,  0.60714286]),\n",
       " array([ 0.60377358,  0.56410256,  0.52      ,  0.54166667,  0.58461538,\n",
       "         0.35555556,  0.59259259,  0.625     ,  0.54166667,  0.48      ]),\n",
       " array([ 0.51282051,  0.25806452,  0.43243243,  0.35      ,  0.20512821,\n",
       "         0.51612903,  0.5       ,  0.47058824,  0.42105263,  0.27027027]),\n",
       " array([ 0.29411765,  0.17391304,  0.24242424,  0.28571429,  0.26315789,\n",
       "         0.2962963 ,  0.14814815,  0.42857143,  0.46666667,  0.06896552]),\n",
       " array([ 0.43902439,  0.55172414,  0.35897436,  0.58536585,  0.33333333,\n",
       "         0.58823529,  0.46153846,  0.52631579,  0.54054054,  0.52380952])]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 实验2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"booster\":\"gbtree\",\n",
    "    \"objective\":\"binary:logistic\",\n",
    "    \"eta\":0.1,\n",
    "    \"max_depth\":10,\n",
    "    \"missing\":0,\n",
    "    \"seed\":0,\n",
    "    \"silent\":1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "betas = [0.1,0.2,0.3,0.4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\teval-error:0.472\ttrain-error:0.425333\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 100 rounds.\n",
      "[1]\teval-error:0.44\ttrain-error:0.406667\n",
      "[2]\teval-error:0.464\ttrain-error:0.422667\n",
      "[3]\teval-error:0.44\ttrain-error:0.406667\n",
      "[4]\teval-error:0.44\ttrain-error:0.406667\n",
      "[5]\teval-error:0.452\ttrain-error:0.405333\n",
      "[6]\teval-error:0.452\ttrain-error:0.405333\n",
      "[7]\teval-error:0.452\ttrain-error:0.404\n",
      "[8]\teval-error:0.452\ttrain-error:0.402667\n",
      "[9]\teval-error:0.46\ttrain-error:0.409333\n",
      "[10]\teval-error:0.456\ttrain-error:0.408\n",
      "[11]\teval-error:0.456\ttrain-error:0.409333\n",
      "[12]\teval-error:0.464\ttrain-error:0.416\n",
      "[13]\teval-error:0.456\ttrain-error:0.410667\n",
      "[14]\teval-error:0.464\ttrain-error:0.417333\n",
      "[15]\teval-error:0.456\ttrain-error:0.414667\n",
      "[16]\teval-error:0.456\ttrain-error:0.414667\n",
      "[17]\teval-error:0.464\ttrain-error:0.416\n",
      "[18]\teval-error:0.456\ttrain-error:0.416\n",
      "[19]\teval-error:0.464\ttrain-error:0.417333\n",
      "[20]\teval-error:0.464\ttrain-error:0.417333\n",
      "[21]\teval-error:0.464\ttrain-error:0.417333\n",
      "[22]\teval-error:0.468\ttrain-error:0.416\n",
      "[23]\teval-error:0.468\ttrain-error:0.416\n",
      "[24]\teval-error:0.468\ttrain-error:0.413333\n",
      "[25]\teval-error:0.476\ttrain-error:0.414667\n",
      "[26]\teval-error:0.48\ttrain-error:0.417333\n",
      "[27]\teval-error:0.476\ttrain-error:0.417333\n",
      "[28]\teval-error:0.48\ttrain-error:0.417333\n",
      "[29]\teval-error:0.48\ttrain-error:0.414667\n",
      "[30]\teval-error:0.48\ttrain-error:0.414667\n",
      "[31]\teval-error:0.48\ttrain-error:0.414667\n",
      "[32]\teval-error:0.48\ttrain-error:0.418667\n",
      "[33]\teval-error:0.48\ttrain-error:0.414667\n",
      "[34]\teval-error:0.484\ttrain-error:0.417333\n",
      "[35]\teval-error:0.48\ttrain-error:0.417333\n",
      "[36]\teval-error:0.484\ttrain-error:0.417333\n",
      "[37]\teval-error:0.484\ttrain-error:0.417333\n",
      "[38]\teval-error:0.48\ttrain-error:0.418667\n",
      "[39]\teval-error:0.48\ttrain-error:0.417333\n",
      "[40]\teval-error:0.48\ttrain-error:0.410667\n",
      "[41]\teval-error:0.48\ttrain-error:0.414667\n",
      "[42]\teval-error:0.448\ttrain-error:0.398667\n",
      "[43]\teval-error:0.464\ttrain-error:0.405333\n",
      "[44]\teval-error:0.464\ttrain-error:0.406667\n",
      "[45]\teval-error:0.464\ttrain-error:0.406667\n",
      "[46]\teval-error:0.46\ttrain-error:0.406667\n",
      "[47]\teval-error:0.46\ttrain-error:0.405333\n",
      "[48]\teval-error:0.46\ttrain-error:0.4\n",
      "[49]\teval-error:0.468\ttrain-error:0.396\n",
      "[50]\teval-error:0.468\ttrain-error:0.394667\n",
      "[51]\teval-error:0.468\ttrain-error:0.393333\n",
      "[52]\teval-error:0.464\ttrain-error:0.389333\n",
      "[53]\teval-error:0.464\ttrain-error:0.390667\n",
      "[54]\teval-error:0.456\ttrain-error:0.388\n",
      "[55]\teval-error:0.46\ttrain-error:0.386667\n",
      "[56]\teval-error:0.46\ttrain-error:0.385333\n",
      "[57]\teval-error:0.46\ttrain-error:0.382667\n",
      "[58]\teval-error:0.46\ttrain-error:0.384\n",
      "[59]\teval-error:0.456\ttrain-error:0.381333\n",
      "[60]\teval-error:0.456\ttrain-error:0.38\n",
      "[61]\teval-error:0.456\ttrain-error:0.38\n",
      "[62]\teval-error:0.452\ttrain-error:0.38\n",
      "[63]\teval-error:0.452\ttrain-error:0.377333\n",
      "[64]\teval-error:0.452\ttrain-error:0.377333\n",
      "[65]\teval-error:0.452\ttrain-error:0.378667\n",
      "[66]\teval-error:0.452\ttrain-error:0.377333\n",
      "[67]\teval-error:0.452\ttrain-error:0.376\n",
      "[68]\teval-error:0.452\ttrain-error:0.376\n",
      "[69]\teval-error:0.452\ttrain-error:0.376\n",
      "[70]\teval-error:0.452\ttrain-error:0.376\n",
      "[71]\teval-error:0.452\ttrain-error:0.374667\n",
      "[72]\teval-error:0.456\ttrain-error:0.373333\n",
      "[73]\teval-error:0.456\ttrain-error:0.373333\n",
      "[74]\teval-error:0.456\ttrain-error:0.374667\n",
      "[75]\teval-error:0.456\ttrain-error:0.373333\n",
      "[76]\teval-error:0.456\ttrain-error:0.374667\n",
      "[77]\teval-error:0.456\ttrain-error:0.374667\n",
      "[78]\teval-error:0.456\ttrain-error:0.374667\n",
      "[79]\teval-error:0.456\ttrain-error:0.374667\n",
      "[80]\teval-error:0.456\ttrain-error:0.373333\n",
      "[81]\teval-error:0.456\ttrain-error:0.369333\n",
      "[82]\teval-error:0.456\ttrain-error:0.368\n",
      "[83]\teval-error:0.456\ttrain-error:0.369333\n",
      "[84]\teval-error:0.456\ttrain-error:0.368\n",
      "[85]\teval-error:0.456\ttrain-error:0.368\n",
      "[86]\teval-error:0.456\ttrain-error:0.368\n",
      "[87]\teval-error:0.456\ttrain-error:0.369333\n",
      "[88]\teval-error:0.456\ttrain-error:0.369333\n",
      "[89]\teval-error:0.456\ttrain-error:0.368\n",
      "[90]\teval-error:0.456\ttrain-error:0.368\n",
      "[91]\teval-error:0.456\ttrain-error:0.368\n",
      "[92]\teval-error:0.456\ttrain-error:0.368\n",
      "[93]\teval-error:0.452\ttrain-error:0.366667\n",
      "[94]\teval-error:0.452\ttrain-error:0.366667\n",
      "[95]\teval-error:0.452\ttrain-error:0.364\n",
      "[96]\teval-error:0.452\ttrain-error:0.364\n",
      "[97]\teval-error:0.452\ttrain-error:0.364\n",
      "[98]\teval-error:0.456\ttrain-error:0.366667\n",
      "[99]\teval-error:0.456\ttrain-error:0.365333\n",
      "[100]\teval-error:0.456\ttrain-error:0.364\n",
      "[101]\teval-error:0.46\ttrain-error:0.366667\n",
      "[102]\teval-error:0.464\ttrain-error:0.370667\n",
      "[103]\teval-error:0.464\ttrain-error:0.369333\n",
      "[104]\teval-error:0.464\ttrain-error:0.370667\n",
      "[105]\teval-error:0.46\ttrain-error:0.362667\n",
      "[106]\teval-error:0.46\ttrain-error:0.362667\n",
      "[107]\teval-error:0.46\ttrain-error:0.364\n",
      "[108]\teval-error:0.46\ttrain-error:0.362667\n",
      "[109]\teval-error:0.456\ttrain-error:0.362667\n",
      "[110]\teval-error:0.452\ttrain-error:0.362667\n",
      "[111]\teval-error:0.452\ttrain-error:0.362667\n",
      "[112]\teval-error:0.452\ttrain-error:0.364\n",
      "[113]\teval-error:0.436\ttrain-error:0.346667\n",
      "[114]\teval-error:0.436\ttrain-error:0.346667\n",
      "[115]\teval-error:0.436\ttrain-error:0.344\n",
      "[116]\teval-error:0.436\ttrain-error:0.344\n",
      "[117]\teval-error:0.436\ttrain-error:0.346667\n",
      "[118]\teval-error:0.436\ttrain-error:0.342667\n",
      "[119]\teval-error:0.436\ttrain-error:0.344\n",
      "[120]\teval-error:0.436\ttrain-error:0.342667\n",
      "[121]\teval-error:0.436\ttrain-error:0.342667\n",
      "[122]\teval-error:0.436\ttrain-error:0.342667\n",
      "[123]\teval-error:0.436\ttrain-error:0.34\n",
      "[124]\teval-error:0.436\ttrain-error:0.341333\n",
      "[125]\teval-error:0.436\ttrain-error:0.34\n",
      "[126]\teval-error:0.436\ttrain-error:0.344\n",
      "[127]\teval-error:0.436\ttrain-error:0.344\n",
      "[128]\teval-error:0.436\ttrain-error:0.342667\n",
      "[129]\teval-error:0.436\ttrain-error:0.344\n",
      "[130]\teval-error:0.436\ttrain-error:0.342667\n",
      "[131]\teval-error:0.436\ttrain-error:0.344\n",
      "[132]\teval-error:0.436\ttrain-error:0.344\n",
      "[133]\teval-error:0.436\ttrain-error:0.344\n",
      "[134]\teval-error:0.436\ttrain-error:0.344\n",
      "[135]\teval-error:0.436\ttrain-error:0.344\n",
      "[136]\teval-error:0.436\ttrain-error:0.344\n",
      "[137]\teval-error:0.436\ttrain-error:0.344\n",
      "[138]\teval-error:0.436\ttrain-error:0.344\n",
      "[139]\teval-error:0.436\ttrain-error:0.344\n",
      "[140]\teval-error:0.436\ttrain-error:0.344\n",
      "[141]\teval-error:0.436\ttrain-error:0.344\n",
      "[142]\teval-error:0.436\ttrain-error:0.344\n",
      "[143]\teval-error:0.436\ttrain-error:0.342667\n",
      "[144]\teval-error:0.436\ttrain-error:0.342667\n",
      "[145]\teval-error:0.44\ttrain-error:0.344\n",
      "[146]\teval-error:0.44\ttrain-error:0.342667\n",
      "[147]\teval-error:0.44\ttrain-error:0.344\n",
      "[148]\teval-error:0.44\ttrain-error:0.344\n",
      "[149]\teval-error:0.44\ttrain-error:0.344\n",
      "[150]\teval-error:0.44\ttrain-error:0.344\n",
      "[151]\teval-error:0.44\ttrain-error:0.344\n",
      "[152]\teval-error:0.44\ttrain-error:0.345333\n",
      "[153]\teval-error:0.44\ttrain-error:0.345333\n",
      "[154]\teval-error:0.44\ttrain-error:0.342667\n",
      "[155]\teval-error:0.44\ttrain-error:0.342667\n",
      "[156]\teval-error:0.44\ttrain-error:0.344\n",
      "[157]\teval-error:0.44\ttrain-error:0.344\n",
      "[158]\teval-error:0.44\ttrain-error:0.342667\n",
      "[159]\teval-error:0.44\ttrain-error:0.342667\n",
      "[160]\teval-error:0.44\ttrain-error:0.342667\n",
      "[161]\teval-error:0.44\ttrain-error:0.342667\n",
      "[162]\teval-error:0.444\ttrain-error:0.334667\n",
      "[163]\teval-error:0.44\ttrain-error:0.341333\n",
      "[164]\teval-error:0.44\ttrain-error:0.341333\n",
      "[165]\teval-error:0.44\ttrain-error:0.341333\n",
      "[166]\teval-error:0.44\ttrain-error:0.342667\n",
      "[167]\teval-error:0.436\ttrain-error:0.345333\n",
      "[168]\teval-error:0.436\ttrain-error:0.344\n",
      "[169]\teval-error:0.436\ttrain-error:0.345333\n",
      "[170]\teval-error:0.44\ttrain-error:0.338667\n",
      "[171]\teval-error:0.436\ttrain-error:0.345333\n",
      "[172]\teval-error:0.436\ttrain-error:0.344\n",
      "[173]\teval-error:0.436\ttrain-error:0.345333\n",
      "[174]\teval-error:0.44\ttrain-error:0.338667\n",
      "[175]\teval-error:0.436\ttrain-error:0.336\n",
      "[176]\teval-error:0.436\ttrain-error:0.336\n",
      "[177]\teval-error:0.436\ttrain-error:0.336\n",
      "[178]\teval-error:0.44\ttrain-error:0.336\n",
      "[179]\teval-error:0.44\ttrain-error:0.336\n",
      "[180]\teval-error:0.436\ttrain-error:0.336\n",
      "[181]\teval-error:0.436\ttrain-error:0.334667\n",
      "[182]\teval-error:0.436\ttrain-error:0.334667\n",
      "[183]\teval-error:0.44\ttrain-error:0.338667\n",
      "[184]\teval-error:0.444\ttrain-error:0.34\n",
      "[185]\teval-error:0.44\ttrain-error:0.338667\n",
      "[186]\teval-error:0.44\ttrain-error:0.34\n",
      "[187]\teval-error:0.444\ttrain-error:0.341333\n",
      "[188]\teval-error:0.44\ttrain-error:0.34\n",
      "[189]\teval-error:0.44\ttrain-error:0.34\n",
      "[190]\teval-error:0.44\ttrain-error:0.34\n",
      "[191]\teval-error:0.44\ttrain-error:0.338667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[192]\teval-error:0.44\ttrain-error:0.34\n",
      "[193]\teval-error:0.44\ttrain-error:0.34\n",
      "[194]\teval-error:0.44\ttrain-error:0.34\n",
      "[195]\teval-error:0.44\ttrain-error:0.34\n",
      "[196]\teval-error:0.44\ttrain-error:0.34\n",
      "[197]\teval-error:0.44\ttrain-error:0.34\n",
      "[198]\teval-error:0.44\ttrain-error:0.34\n",
      "[199]\teval-error:0.44\ttrain-error:0.34\n",
      "[200]\teval-error:0.44\ttrain-error:0.34\n",
      "[201]\teval-error:0.44\ttrain-error:0.34\n",
      "[202]\teval-error:0.44\ttrain-error:0.34\n",
      "[203]\teval-error:0.44\ttrain-error:0.34\n",
      "[204]\teval-error:0.44\ttrain-error:0.338667\n",
      "[205]\teval-error:0.44\ttrain-error:0.338667\n",
      "[206]\teval-error:0.444\ttrain-error:0.338667\n",
      "[207]\teval-error:0.444\ttrain-error:0.338667\n",
      "[208]\teval-error:0.444\ttrain-error:0.337333\n",
      "[209]\teval-error:0.444\ttrain-error:0.337333\n",
      "[210]\teval-error:0.444\ttrain-error:0.337333\n",
      "[211]\teval-error:0.444\ttrain-error:0.337333\n",
      "[212]\teval-error:0.444\ttrain-error:0.337333\n",
      "[213]\teval-error:0.444\ttrain-error:0.337333\n",
      "[214]\teval-error:0.444\ttrain-error:0.338667\n",
      "[215]\teval-error:0.444\ttrain-error:0.338667\n",
      "[216]\teval-error:0.444\ttrain-error:0.337333\n",
      "[217]\teval-error:0.444\ttrain-error:0.337333\n",
      "[218]\teval-error:0.444\ttrain-error:0.337333\n",
      "[219]\teval-error:0.444\ttrain-error:0.338667\n",
      "[220]\teval-error:0.444\ttrain-error:0.337333\n",
      "[221]\teval-error:0.444\ttrain-error:0.337333\n",
      "[222]\teval-error:0.444\ttrain-error:0.337333\n",
      "[223]\teval-error:0.444\ttrain-error:0.337333\n",
      "[224]\teval-error:0.444\ttrain-error:0.337333\n",
      "[225]\teval-error:0.444\ttrain-error:0.337333\n",
      "[226]\teval-error:0.444\ttrain-error:0.337333\n",
      "[227]\teval-error:0.444\ttrain-error:0.337333\n",
      "[228]\teval-error:0.444\ttrain-error:0.337333\n",
      "[229]\teval-error:0.444\ttrain-error:0.337333\n",
      "[230]\teval-error:0.444\ttrain-error:0.337333\n",
      "[231]\teval-error:0.444\ttrain-error:0.337333\n",
      "[232]\teval-error:0.444\ttrain-error:0.337333\n",
      "[233]\teval-error:0.444\ttrain-error:0.337333\n",
      "[234]\teval-error:0.444\ttrain-error:0.337333\n",
      "[235]\teval-error:0.444\ttrain-error:0.337333\n",
      "[236]\teval-error:0.444\ttrain-error:0.337333\n",
      "[237]\teval-error:0.444\ttrain-error:0.337333\n",
      "[238]\teval-error:0.444\ttrain-error:0.337333\n",
      "[239]\teval-error:0.444\ttrain-error:0.337333\n",
      "[240]\teval-error:0.444\ttrain-error:0.337333\n",
      "[241]\teval-error:0.444\ttrain-error:0.337333\n",
      "[242]\teval-error:0.444\ttrain-error:0.337333\n",
      "[243]\teval-error:0.448\ttrain-error:0.337333\n",
      "[244]\teval-error:0.444\ttrain-error:0.337333\n",
      "[245]\teval-error:0.448\ttrain-error:0.337333\n",
      "[246]\teval-error:0.448\ttrain-error:0.337333\n",
      "[247]\teval-error:0.448\ttrain-error:0.337333\n",
      "[248]\teval-error:0.448\ttrain-error:0.336\n",
      "[249]\teval-error:0.448\ttrain-error:0.336\n",
      "[250]\teval-error:0.448\ttrain-error:0.337333\n",
      "[251]\teval-error:0.448\ttrain-error:0.337333\n",
      "[252]\teval-error:0.448\ttrain-error:0.337333\n",
      "[253]\teval-error:0.448\ttrain-error:0.337333\n",
      "[254]\teval-error:0.448\ttrain-error:0.336\n",
      "[255]\teval-error:0.448\ttrain-error:0.337333\n",
      "[256]\teval-error:0.448\ttrain-error:0.337333\n",
      "[257]\teval-error:0.448\ttrain-error:0.336\n",
      "[258]\teval-error:0.448\ttrain-error:0.336\n",
      "[259]\teval-error:0.448\ttrain-error:0.336\n",
      "[260]\teval-error:0.448\ttrain-error:0.336\n",
      "[261]\teval-error:0.448\ttrain-error:0.337333\n",
      "[262]\teval-error:0.448\ttrain-error:0.337333\n",
      "Stopping. Best iteration:\n",
      "[162]\teval-error:0.444\ttrain-error:0.334667\n",
      "\n",
      "[0]\teval-error:0.372\ttrain-error:0.282667\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 100 rounds.\n",
      "[1]\teval-error:0.364\ttrain-error:0.277333\n",
      "[2]\teval-error:0.364\ttrain-error:0.277333\n",
      "[3]\teval-error:0.364\ttrain-error:0.276\n",
      "[4]\teval-error:0.364\ttrain-error:0.276\n",
      "[5]\teval-error:0.368\ttrain-error:0.276\n",
      "[6]\teval-error:0.368\ttrain-error:0.261333\n",
      "[7]\teval-error:0.356\ttrain-error:0.253333\n",
      "[8]\teval-error:0.364\ttrain-error:0.249333\n",
      "[9]\teval-error:0.368\ttrain-error:0.250667\n",
      "[10]\teval-error:0.364\ttrain-error:0.249333\n",
      "[11]\teval-error:0.36\ttrain-error:0.249333\n",
      "[12]\teval-error:0.356\ttrain-error:0.250667\n",
      "[13]\teval-error:0.352\ttrain-error:0.24\n",
      "[14]\teval-error:0.352\ttrain-error:0.242667\n",
      "[15]\teval-error:0.344\ttrain-error:0.233333\n",
      "[16]\teval-error:0.344\ttrain-error:0.234667\n",
      "[17]\teval-error:0.34\ttrain-error:0.233333\n",
      "[18]\teval-error:0.34\ttrain-error:0.232\n",
      "[19]\teval-error:0.34\ttrain-error:0.226667\n",
      "[20]\teval-error:0.34\ttrain-error:0.224\n",
      "[21]\teval-error:0.336\ttrain-error:0.225333\n",
      "[22]\teval-error:0.336\ttrain-error:0.224\n",
      "[23]\teval-error:0.332\ttrain-error:0.221333\n",
      "[24]\teval-error:0.324\ttrain-error:0.218667\n",
      "[25]\teval-error:0.324\ttrain-error:0.217333\n",
      "[26]\teval-error:0.328\ttrain-error:0.217333\n",
      "[27]\teval-error:0.324\ttrain-error:0.209333\n",
      "[28]\teval-error:0.324\ttrain-error:0.210667\n",
      "[29]\teval-error:0.324\ttrain-error:0.213333\n",
      "[30]\teval-error:0.324\ttrain-error:0.210667\n",
      "[31]\teval-error:0.32\ttrain-error:0.209333\n",
      "[32]\teval-error:0.32\ttrain-error:0.208\n",
      "[33]\teval-error:0.32\ttrain-error:0.208\n",
      "[34]\teval-error:0.32\ttrain-error:0.210667\n",
      "[35]\teval-error:0.32\ttrain-error:0.208\n",
      "[36]\teval-error:0.32\ttrain-error:0.208\n",
      "[37]\teval-error:0.32\ttrain-error:0.206667\n",
      "[38]\teval-error:0.324\ttrain-error:0.205333\n",
      "[39]\teval-error:0.324\ttrain-error:0.206667\n",
      "[40]\teval-error:0.324\ttrain-error:0.208\n",
      "[41]\teval-error:0.324\ttrain-error:0.205333\n",
      "[42]\teval-error:0.324\ttrain-error:0.208\n",
      "[43]\teval-error:0.324\ttrain-error:0.204\n",
      "[44]\teval-error:0.324\ttrain-error:0.202667\n",
      "[45]\teval-error:0.324\ttrain-error:0.2\n",
      "[46]\teval-error:0.324\ttrain-error:0.202667\n",
      "[47]\teval-error:0.324\ttrain-error:0.202667\n",
      "[48]\teval-error:0.324\ttrain-error:0.202667\n",
      "[49]\teval-error:0.324\ttrain-error:0.202667\n",
      "[50]\teval-error:0.324\ttrain-error:0.202667\n",
      "[51]\teval-error:0.324\ttrain-error:0.205333\n",
      "[52]\teval-error:0.332\ttrain-error:0.205333\n",
      "[53]\teval-error:0.332\ttrain-error:0.204\n",
      "[54]\teval-error:0.332\ttrain-error:0.206667\n",
      "[55]\teval-error:0.328\ttrain-error:0.202667\n",
      "[56]\teval-error:0.328\ttrain-error:0.202667\n",
      "[57]\teval-error:0.328\ttrain-error:0.201333\n",
      "[58]\teval-error:0.328\ttrain-error:0.204\n",
      "[59]\teval-error:0.328\ttrain-error:0.202667\n",
      "[60]\teval-error:0.336\ttrain-error:0.206667\n",
      "[61]\teval-error:0.336\ttrain-error:0.205333\n",
      "[62]\teval-error:0.34\ttrain-error:0.206667\n",
      "[63]\teval-error:0.332\ttrain-error:0.202667\n",
      "[64]\teval-error:0.34\ttrain-error:0.204\n",
      "[65]\teval-error:0.344\ttrain-error:0.202667\n",
      "[66]\teval-error:0.34\ttrain-error:0.205333\n",
      "[67]\teval-error:0.34\ttrain-error:0.205333\n",
      "[68]\teval-error:0.34\ttrain-error:0.205333\n",
      "[69]\teval-error:0.34\ttrain-error:0.205333\n",
      "[70]\teval-error:0.34\ttrain-error:0.205333\n",
      "[71]\teval-error:0.344\ttrain-error:0.206667\n",
      "[72]\teval-error:0.344\ttrain-error:0.204\n",
      "[73]\teval-error:0.34\ttrain-error:0.202667\n",
      "[74]\teval-error:0.34\ttrain-error:0.202667\n",
      "[75]\teval-error:0.34\ttrain-error:0.202667\n",
      "[76]\teval-error:0.34\ttrain-error:0.202667\n",
      "[77]\teval-error:0.34\ttrain-error:0.202667\n",
      "[78]\teval-error:0.34\ttrain-error:0.204\n",
      "[79]\teval-error:0.344\ttrain-error:0.204\n",
      "[80]\teval-error:0.344\ttrain-error:0.205333\n",
      "[81]\teval-error:0.344\ttrain-error:0.205333\n",
      "[82]\teval-error:0.34\ttrain-error:0.206667\n",
      "[83]\teval-error:0.344\ttrain-error:0.206667\n",
      "[84]\teval-error:0.344\ttrain-error:0.206667\n",
      "[85]\teval-error:0.34\ttrain-error:0.206667\n",
      "[86]\teval-error:0.344\ttrain-error:0.208\n",
      "[87]\teval-error:0.34\ttrain-error:0.205333\n",
      "[88]\teval-error:0.34\ttrain-error:0.206667\n",
      "[89]\teval-error:0.34\ttrain-error:0.205333\n",
      "[90]\teval-error:0.34\ttrain-error:0.205333\n",
      "[91]\teval-error:0.34\ttrain-error:0.204\n",
      "[92]\teval-error:0.34\ttrain-error:0.208\n",
      "[93]\teval-error:0.34\ttrain-error:0.205333\n",
      "[94]\teval-error:0.34\ttrain-error:0.206667\n",
      "[95]\teval-error:0.34\ttrain-error:0.205333\n",
      "[96]\teval-error:0.34\ttrain-error:0.205333\n",
      "[97]\teval-error:0.34\ttrain-error:0.204\n",
      "[98]\teval-error:0.34\ttrain-error:0.204\n",
      "[99]\teval-error:0.34\ttrain-error:0.204\n",
      "[100]\teval-error:0.34\ttrain-error:0.205333\n",
      "[101]\teval-error:0.34\ttrain-error:0.206667\n",
      "[102]\teval-error:0.34\ttrain-error:0.205333\n",
      "[103]\teval-error:0.336\ttrain-error:0.202667\n",
      "[104]\teval-error:0.336\ttrain-error:0.204\n",
      "[105]\teval-error:0.34\ttrain-error:0.204\n",
      "[106]\teval-error:0.332\ttrain-error:0.202667\n",
      "[107]\teval-error:0.332\ttrain-error:0.201333\n",
      "[108]\teval-error:0.332\ttrain-error:0.201333\n",
      "[109]\teval-error:0.332\ttrain-error:0.2\n",
      "[110]\teval-error:0.332\ttrain-error:0.2\n",
      "[111]\teval-error:0.332\ttrain-error:0.202667\n",
      "[112]\teval-error:0.332\ttrain-error:0.202667\n",
      "[113]\teval-error:0.332\ttrain-error:0.201333\n",
      "[114]\teval-error:0.332\ttrain-error:0.201333\n",
      "[115]\teval-error:0.332\ttrain-error:0.2\n",
      "[116]\teval-error:0.332\ttrain-error:0.201333\n",
      "[117]\teval-error:0.332\ttrain-error:0.201333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[118]\teval-error:0.332\ttrain-error:0.201333\n",
      "[119]\teval-error:0.332\ttrain-error:0.201333\n",
      "[120]\teval-error:0.328\ttrain-error:0.201333\n",
      "[121]\teval-error:0.328\ttrain-error:0.201333\n",
      "[122]\teval-error:0.328\ttrain-error:0.201333\n",
      "[123]\teval-error:0.328\ttrain-error:0.201333\n",
      "[124]\teval-error:0.328\ttrain-error:0.202667\n",
      "[125]\teval-error:0.328\ttrain-error:0.201333\n",
      "[126]\teval-error:0.328\ttrain-error:0.201333\n",
      "[127]\teval-error:0.328\ttrain-error:0.201333\n",
      "[128]\teval-error:0.328\ttrain-error:0.201333\n",
      "[129]\teval-error:0.328\ttrain-error:0.202667\n",
      "[130]\teval-error:0.332\ttrain-error:0.201333\n",
      "[131]\teval-error:0.332\ttrain-error:0.2\n",
      "[132]\teval-error:0.332\ttrain-error:0.201333\n",
      "[133]\teval-error:0.332\ttrain-error:0.2\n",
      "[134]\teval-error:0.332\ttrain-error:0.201333\n",
      "[135]\teval-error:0.332\ttrain-error:0.201333\n",
      "[136]\teval-error:0.332\ttrain-error:0.201333\n",
      "[137]\teval-error:0.332\ttrain-error:0.2\n",
      "[138]\teval-error:0.332\ttrain-error:0.2\n",
      "[139]\teval-error:0.332\ttrain-error:0.201333\n",
      "[140]\teval-error:0.336\ttrain-error:0.202667\n",
      "[141]\teval-error:0.336\ttrain-error:0.201333\n",
      "[142]\teval-error:0.336\ttrain-error:0.201333\n",
      "[143]\teval-error:0.336\ttrain-error:0.198667\n",
      "[144]\teval-error:0.336\ttrain-error:0.2\n",
      "[145]\teval-error:0.336\ttrain-error:0.2\n",
      "[146]\teval-error:0.336\ttrain-error:0.198667\n",
      "[147]\teval-error:0.332\ttrain-error:0.198667\n",
      "[148]\teval-error:0.332\ttrain-error:0.2\n",
      "[149]\teval-error:0.332\ttrain-error:0.198667\n",
      "[150]\teval-error:0.332\ttrain-error:0.201333\n",
      "[151]\teval-error:0.328\ttrain-error:0.2\n",
      "[152]\teval-error:0.328\ttrain-error:0.198667\n",
      "[153]\teval-error:0.328\ttrain-error:0.2\n",
      "[154]\teval-error:0.328\ttrain-error:0.2\n",
      "[155]\teval-error:0.328\ttrain-error:0.2\n",
      "[156]\teval-error:0.328\ttrain-error:0.2\n",
      "[157]\teval-error:0.328\ttrain-error:0.2\n",
      "[158]\teval-error:0.328\ttrain-error:0.2\n",
      "[159]\teval-error:0.328\ttrain-error:0.198667\n",
      "[160]\teval-error:0.328\ttrain-error:0.2\n",
      "[161]\teval-error:0.328\ttrain-error:0.201333\n",
      "[162]\teval-error:0.328\ttrain-error:0.2\n",
      "[163]\teval-error:0.328\ttrain-error:0.201333\n",
      "[164]\teval-error:0.332\ttrain-error:0.201333\n",
      "[165]\teval-error:0.332\ttrain-error:0.201333\n",
      "[166]\teval-error:0.332\ttrain-error:0.201333\n",
      "[167]\teval-error:0.332\ttrain-error:0.201333\n",
      "[168]\teval-error:0.332\ttrain-error:0.201333\n",
      "[169]\teval-error:0.332\ttrain-error:0.201333\n",
      "[170]\teval-error:0.332\ttrain-error:0.202667\n",
      "[171]\teval-error:0.332\ttrain-error:0.201333\n",
      "[172]\teval-error:0.332\ttrain-error:0.201333\n",
      "[173]\teval-error:0.332\ttrain-error:0.201333\n",
      "[174]\teval-error:0.332\ttrain-error:0.202667\n",
      "[175]\teval-error:0.332\ttrain-error:0.202667\n",
      "[176]\teval-error:0.332\ttrain-error:0.201333\n",
      "[177]\teval-error:0.332\ttrain-error:0.201333\n",
      "[178]\teval-error:0.332\ttrain-error:0.201333\n",
      "[179]\teval-error:0.332\ttrain-error:0.202667\n",
      "[180]\teval-error:0.332\ttrain-error:0.2\n",
      "[181]\teval-error:0.332\ttrain-error:0.2\n",
      "[182]\teval-error:0.332\ttrain-error:0.2\n",
      "[183]\teval-error:0.332\ttrain-error:0.2\n",
      "[184]\teval-error:0.332\ttrain-error:0.2\n",
      "[185]\teval-error:0.332\ttrain-error:0.2\n",
      "[186]\teval-error:0.332\ttrain-error:0.201333\n",
      "[187]\teval-error:0.332\ttrain-error:0.201333\n",
      "[188]\teval-error:0.328\ttrain-error:0.201333\n",
      "[189]\teval-error:0.332\ttrain-error:0.2\n",
      "[190]\teval-error:0.328\ttrain-error:0.201333\n",
      "[191]\teval-error:0.328\ttrain-error:0.2\n",
      "[192]\teval-error:0.328\ttrain-error:0.2\n",
      "[193]\teval-error:0.328\ttrain-error:0.2\n",
      "[194]\teval-error:0.328\ttrain-error:0.201333\n",
      "[195]\teval-error:0.328\ttrain-error:0.202667\n",
      "[196]\teval-error:0.332\ttrain-error:0.2\n",
      "[197]\teval-error:0.332\ttrain-error:0.202667\n",
      "[198]\teval-error:0.332\ttrain-error:0.2\n",
      "[199]\teval-error:0.332\ttrain-error:0.201333\n",
      "[200]\teval-error:0.336\ttrain-error:0.201333\n",
      "[201]\teval-error:0.332\ttrain-error:0.2\n",
      "[202]\teval-error:0.336\ttrain-error:0.202667\n",
      "[203]\teval-error:0.332\ttrain-error:0.201333\n",
      "[204]\teval-error:0.336\ttrain-error:0.202667\n",
      "[205]\teval-error:0.328\ttrain-error:0.2\n",
      "[206]\teval-error:0.328\ttrain-error:0.2\n",
      "[207]\teval-error:0.328\ttrain-error:0.2\n",
      "[208]\teval-error:0.328\ttrain-error:0.2\n",
      "[209]\teval-error:0.328\ttrain-error:0.201333\n",
      "[210]\teval-error:0.328\ttrain-error:0.2\n",
      "[211]\teval-error:0.328\ttrain-error:0.2\n",
      "[212]\teval-error:0.328\ttrain-error:0.2\n",
      "[213]\teval-error:0.328\ttrain-error:0.2\n",
      "[214]\teval-error:0.328\ttrain-error:0.2\n",
      "[215]\teval-error:0.328\ttrain-error:0.198667\n",
      "[216]\teval-error:0.328\ttrain-error:0.2\n",
      "[217]\teval-error:0.328\ttrain-error:0.2\n",
      "[218]\teval-error:0.328\ttrain-error:0.2\n",
      "[219]\teval-error:0.328\ttrain-error:0.2\n",
      "[220]\teval-error:0.324\ttrain-error:0.2\n",
      "[221]\teval-error:0.328\ttrain-error:0.2\n",
      "[222]\teval-error:0.328\ttrain-error:0.2\n",
      "[223]\teval-error:0.328\ttrain-error:0.2\n",
      "[224]\teval-error:0.328\ttrain-error:0.2\n",
      "[225]\teval-error:0.324\ttrain-error:0.2\n",
      "[226]\teval-error:0.324\ttrain-error:0.2\n",
      "[227]\teval-error:0.324\ttrain-error:0.2\n",
      "[228]\teval-error:0.324\ttrain-error:0.2\n",
      "[229]\teval-error:0.324\ttrain-error:0.2\n",
      "[230]\teval-error:0.324\ttrain-error:0.2\n",
      "[231]\teval-error:0.324\ttrain-error:0.2\n",
      "[232]\teval-error:0.324\ttrain-error:0.2\n",
      "[233]\teval-error:0.324\ttrain-error:0.2\n",
      "[234]\teval-error:0.324\ttrain-error:0.2\n",
      "[235]\teval-error:0.324\ttrain-error:0.2\n",
      "[236]\teval-error:0.328\ttrain-error:0.2\n",
      "[237]\teval-error:0.328\ttrain-error:0.2\n",
      "[238]\teval-error:0.328\ttrain-error:0.2\n",
      "[239]\teval-error:0.328\ttrain-error:0.2\n",
      "[240]\teval-error:0.328\ttrain-error:0.2\n",
      "[241]\teval-error:0.328\ttrain-error:0.2\n",
      "[242]\teval-error:0.328\ttrain-error:0.2\n",
      "[243]\teval-error:0.328\ttrain-error:0.2\n",
      "Stopping. Best iteration:\n",
      "[143]\teval-error:0.336\ttrain-error:0.198667\n",
      "\n",
      "[0]\teval-error:0.328\ttrain-error:0.226667\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 100 rounds.\n",
      "[1]\teval-error:0.328\ttrain-error:0.221333\n",
      "[2]\teval-error:0.328\ttrain-error:0.221333\n",
      "[3]\teval-error:0.328\ttrain-error:0.218667\n",
      "[4]\teval-error:0.328\ttrain-error:0.213333\n",
      "[5]\teval-error:0.328\ttrain-error:0.213333\n",
      "[6]\teval-error:0.328\ttrain-error:0.193333\n",
      "[7]\teval-error:0.344\ttrain-error:0.192\n",
      "[8]\teval-error:0.344\ttrain-error:0.188\n",
      "[9]\teval-error:0.344\ttrain-error:0.185333\n",
      "[10]\teval-error:0.34\ttrain-error:0.184\n",
      "[11]\teval-error:0.34\ttrain-error:0.184\n",
      "[12]\teval-error:0.328\ttrain-error:0.178667\n",
      "[13]\teval-error:0.324\ttrain-error:0.169333\n",
      "[14]\teval-error:0.32\ttrain-error:0.170667\n",
      "[15]\teval-error:0.324\ttrain-error:0.168\n",
      "[16]\teval-error:0.32\ttrain-error:0.170667\n",
      "[17]\teval-error:0.324\ttrain-error:0.166667\n",
      "[18]\teval-error:0.324\ttrain-error:0.164\n",
      "[19]\teval-error:0.332\ttrain-error:0.165333\n",
      "[20]\teval-error:0.332\ttrain-error:0.165333\n",
      "[21]\teval-error:0.332\ttrain-error:0.162667\n",
      "[22]\teval-error:0.336\ttrain-error:0.161333\n",
      "[23]\teval-error:0.34\ttrain-error:0.161333\n",
      "[24]\teval-error:0.34\ttrain-error:0.16\n",
      "[25]\teval-error:0.336\ttrain-error:0.16\n",
      "[26]\teval-error:0.328\ttrain-error:0.162667\n",
      "[27]\teval-error:0.336\ttrain-error:0.161333\n",
      "[28]\teval-error:0.34\ttrain-error:0.162667\n",
      "[29]\teval-error:0.336\ttrain-error:0.165333\n",
      "[30]\teval-error:0.336\ttrain-error:0.165333\n",
      "[31]\teval-error:0.336\ttrain-error:0.165333\n",
      "[32]\teval-error:0.336\ttrain-error:0.165333\n",
      "[33]\teval-error:0.332\ttrain-error:0.162667\n",
      "[34]\teval-error:0.332\ttrain-error:0.164\n",
      "[35]\teval-error:0.332\ttrain-error:0.164\n",
      "[36]\teval-error:0.328\ttrain-error:0.161333\n",
      "[37]\teval-error:0.324\ttrain-error:0.158667\n",
      "[38]\teval-error:0.324\ttrain-error:0.157333\n",
      "[39]\teval-error:0.328\ttrain-error:0.16\n",
      "[40]\teval-error:0.324\ttrain-error:0.157333\n",
      "[41]\teval-error:0.324\ttrain-error:0.156\n",
      "[42]\teval-error:0.32\ttrain-error:0.156\n",
      "[43]\teval-error:0.32\ttrain-error:0.156\n",
      "[44]\teval-error:0.324\ttrain-error:0.154667\n",
      "[45]\teval-error:0.324\ttrain-error:0.154667\n",
      "[46]\teval-error:0.32\ttrain-error:0.153333\n",
      "[47]\teval-error:0.32\ttrain-error:0.153333\n",
      "[48]\teval-error:0.32\ttrain-error:0.154667\n",
      "[49]\teval-error:0.32\ttrain-error:0.152\n",
      "[50]\teval-error:0.32\ttrain-error:0.156\n",
      "[51]\teval-error:0.32\ttrain-error:0.153333\n",
      "[52]\teval-error:0.32\ttrain-error:0.153333\n",
      "[53]\teval-error:0.32\ttrain-error:0.153333\n",
      "[54]\teval-error:0.32\ttrain-error:0.153333\n",
      "[55]\teval-error:0.32\ttrain-error:0.157333\n",
      "[56]\teval-error:0.32\ttrain-error:0.158667\n",
      "[57]\teval-error:0.32\ttrain-error:0.157333\n",
      "[58]\teval-error:0.32\ttrain-error:0.157333\n",
      "[59]\teval-error:0.32\ttrain-error:0.156\n",
      "[60]\teval-error:0.32\ttrain-error:0.156\n",
      "[61]\teval-error:0.316\ttrain-error:0.154667\n",
      "[62]\teval-error:0.312\ttrain-error:0.156\n",
      "[63]\teval-error:0.316\ttrain-error:0.154667\n",
      "[64]\teval-error:0.316\ttrain-error:0.154667\n",
      "[65]\teval-error:0.312\ttrain-error:0.156\n",
      "[66]\teval-error:0.308\ttrain-error:0.154667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[67]\teval-error:0.308\ttrain-error:0.153333\n",
      "[68]\teval-error:0.308\ttrain-error:0.156\n",
      "[69]\teval-error:0.312\ttrain-error:0.154667\n",
      "[70]\teval-error:0.308\ttrain-error:0.154667\n",
      "[71]\teval-error:0.312\ttrain-error:0.154667\n",
      "[72]\teval-error:0.312\ttrain-error:0.153333\n",
      "[73]\teval-error:0.308\ttrain-error:0.150667\n",
      "[74]\teval-error:0.312\ttrain-error:0.154667\n",
      "[75]\teval-error:0.312\ttrain-error:0.157333\n",
      "[76]\teval-error:0.312\ttrain-error:0.154667\n",
      "[77]\teval-error:0.312\ttrain-error:0.154667\n",
      "[78]\teval-error:0.308\ttrain-error:0.154667\n",
      "[79]\teval-error:0.308\ttrain-error:0.153333\n",
      "[80]\teval-error:0.308\ttrain-error:0.153333\n",
      "[81]\teval-error:0.304\ttrain-error:0.154667\n",
      "[82]\teval-error:0.308\ttrain-error:0.154667\n",
      "[83]\teval-error:0.308\ttrain-error:0.154667\n",
      "[84]\teval-error:0.308\ttrain-error:0.154667\n",
      "[85]\teval-error:0.304\ttrain-error:0.156\n",
      "[86]\teval-error:0.304\ttrain-error:0.154667\n",
      "[87]\teval-error:0.312\ttrain-error:0.156\n",
      "[88]\teval-error:0.312\ttrain-error:0.156\n",
      "[89]\teval-error:0.304\ttrain-error:0.154667\n",
      "[90]\teval-error:0.312\ttrain-error:0.154667\n",
      "[91]\teval-error:0.308\ttrain-error:0.154667\n",
      "[92]\teval-error:0.308\ttrain-error:0.154667\n",
      "[93]\teval-error:0.3\ttrain-error:0.157333\n",
      "[94]\teval-error:0.308\ttrain-error:0.157333\n",
      "[95]\teval-error:0.304\ttrain-error:0.158667\n",
      "[96]\teval-error:0.304\ttrain-error:0.158667\n",
      "[97]\teval-error:0.3\ttrain-error:0.158667\n",
      "[98]\teval-error:0.308\ttrain-error:0.157333\n",
      "[99]\teval-error:0.304\ttrain-error:0.157333\n",
      "[100]\teval-error:0.3\ttrain-error:0.158667\n",
      "[101]\teval-error:0.3\ttrain-error:0.158667\n",
      "[102]\teval-error:0.304\ttrain-error:0.16\n",
      "[103]\teval-error:0.304\ttrain-error:0.16\n",
      "[104]\teval-error:0.3\ttrain-error:0.16\n",
      "[105]\teval-error:0.304\ttrain-error:0.16\n",
      "[106]\teval-error:0.308\ttrain-error:0.158667\n",
      "[107]\teval-error:0.308\ttrain-error:0.16\n",
      "[108]\teval-error:0.308\ttrain-error:0.158667\n",
      "[109]\teval-error:0.304\ttrain-error:0.158667\n",
      "[110]\teval-error:0.308\ttrain-error:0.158667\n",
      "[111]\teval-error:0.308\ttrain-error:0.158667\n",
      "[112]\teval-error:0.308\ttrain-error:0.16\n",
      "[113]\teval-error:0.304\ttrain-error:0.158667\n",
      "[114]\teval-error:0.308\ttrain-error:0.158667\n",
      "[115]\teval-error:0.304\ttrain-error:0.158667\n",
      "[116]\teval-error:0.304\ttrain-error:0.158667\n",
      "[117]\teval-error:0.304\ttrain-error:0.158667\n",
      "[118]\teval-error:0.308\ttrain-error:0.158667\n",
      "[119]\teval-error:0.304\ttrain-error:0.154667\n",
      "[120]\teval-error:0.304\ttrain-error:0.156\n",
      "[121]\teval-error:0.304\ttrain-error:0.153333\n",
      "[122]\teval-error:0.308\ttrain-error:0.153333\n",
      "[123]\teval-error:0.304\ttrain-error:0.153333\n",
      "[124]\teval-error:0.304\ttrain-error:0.153333\n",
      "[125]\teval-error:0.308\ttrain-error:0.153333\n",
      "[126]\teval-error:0.308\ttrain-error:0.153333\n",
      "[127]\teval-error:0.304\ttrain-error:0.154667\n",
      "[128]\teval-error:0.304\ttrain-error:0.154667\n",
      "[129]\teval-error:0.308\ttrain-error:0.156\n",
      "[130]\teval-error:0.308\ttrain-error:0.154667\n",
      "[131]\teval-error:0.308\ttrain-error:0.154667\n",
      "[132]\teval-error:0.308\ttrain-error:0.154667\n",
      "[133]\teval-error:0.316\ttrain-error:0.153333\n",
      "[134]\teval-error:0.316\ttrain-error:0.153333\n",
      "[135]\teval-error:0.312\ttrain-error:0.152\n",
      "[136]\teval-error:0.312\ttrain-error:0.153333\n",
      "[137]\teval-error:0.308\ttrain-error:0.153333\n",
      "[138]\teval-error:0.312\ttrain-error:0.153333\n",
      "[139]\teval-error:0.312\ttrain-error:0.153333\n",
      "[140]\teval-error:0.308\ttrain-error:0.152\n",
      "[141]\teval-error:0.312\ttrain-error:0.152\n",
      "[142]\teval-error:0.308\ttrain-error:0.149333\n",
      "[143]\teval-error:0.304\ttrain-error:0.152\n",
      "[144]\teval-error:0.308\ttrain-error:0.150667\n",
      "[145]\teval-error:0.304\ttrain-error:0.150667\n",
      "[146]\teval-error:0.304\ttrain-error:0.150667\n",
      "[147]\teval-error:0.308\ttrain-error:0.150667\n",
      "[148]\teval-error:0.304\ttrain-error:0.149333\n",
      "[149]\teval-error:0.304\ttrain-error:0.149333\n",
      "[150]\teval-error:0.304\ttrain-error:0.150667\n",
      "[151]\teval-error:0.3\ttrain-error:0.150667\n",
      "[152]\teval-error:0.304\ttrain-error:0.152\n",
      "[153]\teval-error:0.304\ttrain-error:0.150667\n",
      "[154]\teval-error:0.3\ttrain-error:0.150667\n",
      "[155]\teval-error:0.3\ttrain-error:0.150667\n",
      "[156]\teval-error:0.304\ttrain-error:0.149333\n",
      "[157]\teval-error:0.3\ttrain-error:0.149333\n",
      "[158]\teval-error:0.3\ttrain-error:0.149333\n",
      "[159]\teval-error:0.304\ttrain-error:0.148\n",
      "[160]\teval-error:0.3\ttrain-error:0.149333\n",
      "[161]\teval-error:0.3\ttrain-error:0.149333\n",
      "[162]\teval-error:0.304\ttrain-error:0.148\n",
      "[163]\teval-error:0.304\ttrain-error:0.149333\n",
      "[164]\teval-error:0.304\ttrain-error:0.149333\n",
      "[165]\teval-error:0.304\ttrain-error:0.148\n",
      "[166]\teval-error:0.304\ttrain-error:0.149333\n",
      "[167]\teval-error:0.304\ttrain-error:0.148\n",
      "[168]\teval-error:0.304\ttrain-error:0.148\n",
      "[169]\teval-error:0.304\ttrain-error:0.148\n",
      "[170]\teval-error:0.304\ttrain-error:0.146667\n",
      "[171]\teval-error:0.3\ttrain-error:0.146667\n",
      "[172]\teval-error:0.3\ttrain-error:0.146667\n",
      "[173]\teval-error:0.3\ttrain-error:0.148\n",
      "[174]\teval-error:0.3\ttrain-error:0.146667\n",
      "[175]\teval-error:0.3\ttrain-error:0.146667\n",
      "[176]\teval-error:0.3\ttrain-error:0.144\n",
      "[177]\teval-error:0.3\ttrain-error:0.144\n",
      "[178]\teval-error:0.3\ttrain-error:0.145333\n",
      "[179]\teval-error:0.3\ttrain-error:0.142667\n",
      "[180]\teval-error:0.3\ttrain-error:0.144\n",
      "[181]\teval-error:0.3\ttrain-error:0.144\n",
      "[182]\teval-error:0.3\ttrain-error:0.144\n",
      "[183]\teval-error:0.3\ttrain-error:0.142667\n",
      "[184]\teval-error:0.3\ttrain-error:0.142667\n",
      "[185]\teval-error:0.3\ttrain-error:0.142667\n",
      "[186]\teval-error:0.3\ttrain-error:0.141333\n",
      "[187]\teval-error:0.3\ttrain-error:0.141333\n",
      "[188]\teval-error:0.3\ttrain-error:0.141333\n",
      "[189]\teval-error:0.296\ttrain-error:0.141333\n",
      "[190]\teval-error:0.3\ttrain-error:0.141333\n",
      "[191]\teval-error:0.296\ttrain-error:0.14\n",
      "[192]\teval-error:0.296\ttrain-error:0.141333\n",
      "[193]\teval-error:0.296\ttrain-error:0.141333\n",
      "[194]\teval-error:0.296\ttrain-error:0.141333\n",
      "[195]\teval-error:0.296\ttrain-error:0.14\n",
      "[196]\teval-error:0.296\ttrain-error:0.141333\n",
      "[197]\teval-error:0.296\ttrain-error:0.141333\n",
      "[198]\teval-error:0.296\ttrain-error:0.141333\n",
      "[199]\teval-error:0.296\ttrain-error:0.141333\n",
      "[200]\teval-error:0.296\ttrain-error:0.145333\n",
      "[201]\teval-error:0.296\ttrain-error:0.141333\n",
      "[202]\teval-error:0.296\ttrain-error:0.142667\n",
      "[203]\teval-error:0.296\ttrain-error:0.141333\n",
      "[204]\teval-error:0.296\ttrain-error:0.141333\n",
      "[205]\teval-error:0.296\ttrain-error:0.141333\n",
      "[206]\teval-error:0.296\ttrain-error:0.141333\n",
      "[207]\teval-error:0.296\ttrain-error:0.141333\n",
      "[208]\teval-error:0.296\ttrain-error:0.141333\n",
      "[209]\teval-error:0.296\ttrain-error:0.142667\n",
      "[210]\teval-error:0.296\ttrain-error:0.142667\n",
      "[211]\teval-error:0.296\ttrain-error:0.142667\n",
      "[212]\teval-error:0.296\ttrain-error:0.142667\n",
      "[213]\teval-error:0.296\ttrain-error:0.142667\n",
      "[214]\teval-error:0.296\ttrain-error:0.141333\n",
      "[215]\teval-error:0.296\ttrain-error:0.142667\n",
      "[216]\teval-error:0.296\ttrain-error:0.141333\n",
      "[217]\teval-error:0.3\ttrain-error:0.142667\n",
      "[218]\teval-error:0.3\ttrain-error:0.142667\n",
      "[219]\teval-error:0.296\ttrain-error:0.142667\n",
      "[220]\teval-error:0.292\ttrain-error:0.142667\n",
      "[221]\teval-error:0.292\ttrain-error:0.142667\n",
      "[222]\teval-error:0.292\ttrain-error:0.141333\n",
      "[223]\teval-error:0.292\ttrain-error:0.141333\n",
      "[224]\teval-error:0.284\ttrain-error:0.142667\n",
      "[225]\teval-error:0.284\ttrain-error:0.141333\n",
      "[226]\teval-error:0.284\ttrain-error:0.141333\n",
      "[227]\teval-error:0.284\ttrain-error:0.142667\n",
      "[228]\teval-error:0.284\ttrain-error:0.142667\n",
      "[229]\teval-error:0.284\ttrain-error:0.141333\n",
      "[230]\teval-error:0.284\ttrain-error:0.14\n",
      "[231]\teval-error:0.284\ttrain-error:0.138667\n",
      "[232]\teval-error:0.284\ttrain-error:0.137333\n",
      "[233]\teval-error:0.284\ttrain-error:0.137333\n",
      "[234]\teval-error:0.284\ttrain-error:0.137333\n",
      "[235]\teval-error:0.284\ttrain-error:0.137333\n",
      "[236]\teval-error:0.284\ttrain-error:0.137333\n",
      "[237]\teval-error:0.284\ttrain-error:0.137333\n",
      "[238]\teval-error:0.284\ttrain-error:0.137333\n",
      "[239]\teval-error:0.284\ttrain-error:0.137333\n",
      "[240]\teval-error:0.284\ttrain-error:0.138667\n",
      "[241]\teval-error:0.284\ttrain-error:0.14\n",
      "[242]\teval-error:0.284\ttrain-error:0.14\n",
      "[243]\teval-error:0.284\ttrain-error:0.14\n",
      "[244]\teval-error:0.284\ttrain-error:0.138667\n",
      "[245]\teval-error:0.284\ttrain-error:0.14\n",
      "[246]\teval-error:0.284\ttrain-error:0.138667\n",
      "[247]\teval-error:0.284\ttrain-error:0.137333\n",
      "[248]\teval-error:0.284\ttrain-error:0.137333\n",
      "[249]\teval-error:0.284\ttrain-error:0.136\n",
      "[250]\teval-error:0.284\ttrain-error:0.137333\n",
      "[251]\teval-error:0.284\ttrain-error:0.137333\n",
      "[252]\teval-error:0.284\ttrain-error:0.137333\n",
      "[253]\teval-error:0.284\ttrain-error:0.137333\n",
      "[254]\teval-error:0.284\ttrain-error:0.136\n",
      "[255]\teval-error:0.284\ttrain-error:0.136\n",
      "[256]\teval-error:0.284\ttrain-error:0.134667\n",
      "[257]\teval-error:0.284\ttrain-error:0.136\n",
      "[258]\teval-error:0.284\ttrain-error:0.136\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[259]\teval-error:0.284\ttrain-error:0.136\n",
      "[260]\teval-error:0.284\ttrain-error:0.136\n",
      "[261]\teval-error:0.284\ttrain-error:0.136\n",
      "[262]\teval-error:0.288\ttrain-error:0.136\n",
      "[263]\teval-error:0.288\ttrain-error:0.136\n",
      "[264]\teval-error:0.288\ttrain-error:0.136\n",
      "[265]\teval-error:0.284\ttrain-error:0.133333\n",
      "[266]\teval-error:0.284\ttrain-error:0.134667\n",
      "[267]\teval-error:0.288\ttrain-error:0.134667\n",
      "[268]\teval-error:0.288\ttrain-error:0.134667\n",
      "[269]\teval-error:0.288\ttrain-error:0.134667\n",
      "[270]\teval-error:0.288\ttrain-error:0.133333\n",
      "[271]\teval-error:0.288\ttrain-error:0.136\n",
      "[272]\teval-error:0.288\ttrain-error:0.136\n",
      "[273]\teval-error:0.292\ttrain-error:0.136\n",
      "[274]\teval-error:0.292\ttrain-error:0.136\n",
      "[275]\teval-error:0.292\ttrain-error:0.137333\n",
      "[276]\teval-error:0.292\ttrain-error:0.137333\n",
      "[277]\teval-error:0.288\ttrain-error:0.137333\n",
      "[278]\teval-error:0.288\ttrain-error:0.137333\n",
      "[279]\teval-error:0.288\ttrain-error:0.136\n",
      "[280]\teval-error:0.288\ttrain-error:0.137333\n",
      "[281]\teval-error:0.292\ttrain-error:0.134667\n",
      "[282]\teval-error:0.292\ttrain-error:0.136\n",
      "[283]\teval-error:0.292\ttrain-error:0.137333\n",
      "[284]\teval-error:0.292\ttrain-error:0.134667\n",
      "[285]\teval-error:0.292\ttrain-error:0.133333\n",
      "[286]\teval-error:0.292\ttrain-error:0.133333\n",
      "[287]\teval-error:0.292\ttrain-error:0.136\n",
      "[288]\teval-error:0.292\ttrain-error:0.138667\n",
      "[289]\teval-error:0.292\ttrain-error:0.136\n",
      "[290]\teval-error:0.292\ttrain-error:0.138667\n",
      "[291]\teval-error:0.292\ttrain-error:0.134667\n",
      "[292]\teval-error:0.292\ttrain-error:0.134667\n",
      "[293]\teval-error:0.292\ttrain-error:0.134667\n",
      "[294]\teval-error:0.292\ttrain-error:0.133333\n",
      "[295]\teval-error:0.292\ttrain-error:0.133333\n",
      "[296]\teval-error:0.292\ttrain-error:0.132\n",
      "[297]\teval-error:0.292\ttrain-error:0.132\n",
      "[298]\teval-error:0.292\ttrain-error:0.132\n",
      "[299]\teval-error:0.292\ttrain-error:0.133333\n",
      "[300]\teval-error:0.292\ttrain-error:0.134667\n",
      "[301]\teval-error:0.292\ttrain-error:0.133333\n",
      "[302]\teval-error:0.292\ttrain-error:0.133333\n",
      "[303]\teval-error:0.292\ttrain-error:0.133333\n",
      "[304]\teval-error:0.292\ttrain-error:0.132\n",
      "[305]\teval-error:0.288\ttrain-error:0.133333\n",
      "[306]\teval-error:0.284\ttrain-error:0.132\n",
      "[307]\teval-error:0.288\ttrain-error:0.133333\n",
      "[308]\teval-error:0.284\ttrain-error:0.136\n",
      "[309]\teval-error:0.284\ttrain-error:0.132\n",
      "[310]\teval-error:0.284\ttrain-error:0.133333\n",
      "[311]\teval-error:0.288\ttrain-error:0.132\n",
      "[312]\teval-error:0.288\ttrain-error:0.132\n",
      "[313]\teval-error:0.288\ttrain-error:0.132\n",
      "[314]\teval-error:0.288\ttrain-error:0.130667\n",
      "[315]\teval-error:0.288\ttrain-error:0.134667\n",
      "[316]\teval-error:0.288\ttrain-error:0.133333\n",
      "[317]\teval-error:0.288\ttrain-error:0.134667\n",
      "[318]\teval-error:0.288\ttrain-error:0.133333\n",
      "[319]\teval-error:0.288\ttrain-error:0.133333\n",
      "[320]\teval-error:0.288\ttrain-error:0.133333\n",
      "[321]\teval-error:0.288\ttrain-error:0.134667\n",
      "[322]\teval-error:0.288\ttrain-error:0.133333\n",
      "[323]\teval-error:0.288\ttrain-error:0.134667\n",
      "[324]\teval-error:0.288\ttrain-error:0.134667\n",
      "[325]\teval-error:0.288\ttrain-error:0.133333\n",
      "[326]\teval-error:0.288\ttrain-error:0.134667\n",
      "[327]\teval-error:0.288\ttrain-error:0.134667\n",
      "[328]\teval-error:0.288\ttrain-error:0.134667\n",
      "[329]\teval-error:0.288\ttrain-error:0.134667\n",
      "[330]\teval-error:0.288\ttrain-error:0.136\n",
      "[331]\teval-error:0.288\ttrain-error:0.134667\n",
      "[332]\teval-error:0.288\ttrain-error:0.136\n",
      "[333]\teval-error:0.292\ttrain-error:0.132\n",
      "[334]\teval-error:0.292\ttrain-error:0.134667\n",
      "[335]\teval-error:0.296\ttrain-error:0.129333\n",
      "[336]\teval-error:0.296\ttrain-error:0.132\n",
      "[337]\teval-error:0.296\ttrain-error:0.132\n",
      "[338]\teval-error:0.296\ttrain-error:0.133333\n",
      "[339]\teval-error:0.296\ttrain-error:0.133333\n",
      "[340]\teval-error:0.296\ttrain-error:0.133333\n",
      "[341]\teval-error:0.296\ttrain-error:0.133333\n",
      "[342]\teval-error:0.296\ttrain-error:0.133333\n",
      "[343]\teval-error:0.296\ttrain-error:0.133333\n",
      "[344]\teval-error:0.296\ttrain-error:0.133333\n",
      "[345]\teval-error:0.296\ttrain-error:0.129333\n",
      "[346]\teval-error:0.296\ttrain-error:0.133333\n",
      "[347]\teval-error:0.296\ttrain-error:0.133333\n",
      "[348]\teval-error:0.296\ttrain-error:0.129333\n",
      "[349]\teval-error:0.296\ttrain-error:0.133333\n",
      "[350]\teval-error:0.296\ttrain-error:0.129333\n",
      "[351]\teval-error:0.296\ttrain-error:0.128\n",
      "[352]\teval-error:0.296\ttrain-error:0.132\n",
      "[353]\teval-error:0.296\ttrain-error:0.133333\n",
      "[354]\teval-error:0.296\ttrain-error:0.132\n",
      "[355]\teval-error:0.296\ttrain-error:0.132\n",
      "[356]\teval-error:0.3\ttrain-error:0.133333\n",
      "[357]\teval-error:0.3\ttrain-error:0.133333\n",
      "[358]\teval-error:0.3\ttrain-error:0.133333\n",
      "[359]\teval-error:0.3\ttrain-error:0.134667\n",
      "[360]\teval-error:0.3\ttrain-error:0.130667\n",
      "[361]\teval-error:0.3\ttrain-error:0.133333\n",
      "[362]\teval-error:0.3\ttrain-error:0.133333\n",
      "[363]\teval-error:0.3\ttrain-error:0.129333\n",
      "[364]\teval-error:0.3\ttrain-error:0.133333\n",
      "[365]\teval-error:0.3\ttrain-error:0.129333\n",
      "[366]\teval-error:0.3\ttrain-error:0.129333\n",
      "[367]\teval-error:0.296\ttrain-error:0.129333\n",
      "[368]\teval-error:0.3\ttrain-error:0.133333\n",
      "[369]\teval-error:0.3\ttrain-error:0.133333\n",
      "[370]\teval-error:0.3\ttrain-error:0.130667\n",
      "[371]\teval-error:0.3\ttrain-error:0.134667\n",
      "[372]\teval-error:0.296\ttrain-error:0.130667\n",
      "[373]\teval-error:0.3\ttrain-error:0.129333\n",
      "[374]\teval-error:0.296\ttrain-error:0.133333\n",
      "[375]\teval-error:0.296\ttrain-error:0.129333\n",
      "[376]\teval-error:0.3\ttrain-error:0.129333\n",
      "[377]\teval-error:0.296\ttrain-error:0.129333\n",
      "[378]\teval-error:0.296\ttrain-error:0.129333\n",
      "[379]\teval-error:0.296\ttrain-error:0.129333\n",
      "[380]\teval-error:0.292\ttrain-error:0.130667\n",
      "[381]\teval-error:0.296\ttrain-error:0.132\n",
      "[382]\teval-error:0.296\ttrain-error:0.132\n",
      "[383]\teval-error:0.292\ttrain-error:0.132\n",
      "[384]\teval-error:0.296\ttrain-error:0.132\n",
      "[385]\teval-error:0.296\ttrain-error:0.132\n",
      "[386]\teval-error:0.292\ttrain-error:0.136\n",
      "[387]\teval-error:0.296\ttrain-error:0.132\n",
      "[388]\teval-error:0.292\ttrain-error:0.132\n",
      "[389]\teval-error:0.292\ttrain-error:0.134667\n",
      "[390]\teval-error:0.292\ttrain-error:0.130667\n",
      "[391]\teval-error:0.292\ttrain-error:0.132\n",
      "[392]\teval-error:0.292\ttrain-error:0.132\n",
      "[393]\teval-error:0.292\ttrain-error:0.132\n",
      "[394]\teval-error:0.292\ttrain-error:0.130667\n",
      "[395]\teval-error:0.292\ttrain-error:0.130667\n",
      "[396]\teval-error:0.292\ttrain-error:0.132\n",
      "[397]\teval-error:0.292\ttrain-error:0.132\n",
      "[398]\teval-error:0.292\ttrain-error:0.132\n",
      "[399]\teval-error:0.292\ttrain-error:0.132\n",
      "[400]\teval-error:0.292\ttrain-error:0.132\n",
      "[401]\teval-error:0.292\ttrain-error:0.134667\n",
      "[402]\teval-error:0.292\ttrain-error:0.130667\n",
      "[403]\teval-error:0.292\ttrain-error:0.130667\n",
      "[404]\teval-error:0.292\ttrain-error:0.132\n",
      "[405]\teval-error:0.292\ttrain-error:0.129333\n",
      "[406]\teval-error:0.292\ttrain-error:0.130667\n",
      "[407]\teval-error:0.292\ttrain-error:0.130667\n",
      "[408]\teval-error:0.292\ttrain-error:0.130667\n",
      "[409]\teval-error:0.292\ttrain-error:0.132\n",
      "[410]\teval-error:0.292\ttrain-error:0.130667\n",
      "[411]\teval-error:0.292\ttrain-error:0.130667\n",
      "[412]\teval-error:0.292\ttrain-error:0.132\n",
      "[413]\teval-error:0.292\ttrain-error:0.132\n",
      "[414]\teval-error:0.292\ttrain-error:0.132\n",
      "[415]\teval-error:0.292\ttrain-error:0.130667\n",
      "[416]\teval-error:0.292\ttrain-error:0.132\n",
      "[417]\teval-error:0.292\ttrain-error:0.132\n",
      "[418]\teval-error:0.292\ttrain-error:0.130667\n",
      "[419]\teval-error:0.292\ttrain-error:0.132\n",
      "[420]\teval-error:0.292\ttrain-error:0.130667\n",
      "[421]\teval-error:0.292\ttrain-error:0.130667\n",
      "[422]\teval-error:0.292\ttrain-error:0.130667\n",
      "[423]\teval-error:0.292\ttrain-error:0.130667\n",
      "[424]\teval-error:0.292\ttrain-error:0.130667\n",
      "[425]\teval-error:0.292\ttrain-error:0.130667\n",
      "[426]\teval-error:0.296\ttrain-error:0.129333\n",
      "[427]\teval-error:0.296\ttrain-error:0.129333\n",
      "[428]\teval-error:0.296\ttrain-error:0.128\n",
      "[429]\teval-error:0.296\ttrain-error:0.128\n",
      "[430]\teval-error:0.296\ttrain-error:0.128\n",
      "[431]\teval-error:0.296\ttrain-error:0.126667\n",
      "[432]\teval-error:0.292\ttrain-error:0.128\n",
      "[433]\teval-error:0.292\ttrain-error:0.128\n",
      "[434]\teval-error:0.292\ttrain-error:0.128\n",
      "[435]\teval-error:0.288\ttrain-error:0.126667\n",
      "[436]\teval-error:0.288\ttrain-error:0.125333\n",
      "[437]\teval-error:0.288\ttrain-error:0.125333\n",
      "[438]\teval-error:0.288\ttrain-error:0.125333\n",
      "[439]\teval-error:0.288\ttrain-error:0.125333\n",
      "[440]\teval-error:0.288\ttrain-error:0.125333\n",
      "[441]\teval-error:0.288\ttrain-error:0.125333\n",
      "[442]\teval-error:0.288\ttrain-error:0.124\n",
      "[443]\teval-error:0.288\ttrain-error:0.124\n",
      "[444]\teval-error:0.288\ttrain-error:0.124\n",
      "[445]\teval-error:0.288\ttrain-error:0.125333\n",
      "[446]\teval-error:0.288\ttrain-error:0.125333\n",
      "[447]\teval-error:0.288\ttrain-error:0.124\n",
      "[448]\teval-error:0.288\ttrain-error:0.124\n",
      "[449]\teval-error:0.288\ttrain-error:0.125333\n",
      "[450]\teval-error:0.288\ttrain-error:0.125333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[451]\teval-error:0.288\ttrain-error:0.125333\n",
      "[452]\teval-error:0.288\ttrain-error:0.125333\n",
      "[453]\teval-error:0.288\ttrain-error:0.125333\n",
      "[454]\teval-error:0.288\ttrain-error:0.125333\n",
      "[455]\teval-error:0.288\ttrain-error:0.125333\n",
      "[456]\teval-error:0.288\ttrain-error:0.125333\n",
      "[457]\teval-error:0.288\ttrain-error:0.125333\n",
      "[458]\teval-error:0.288\ttrain-error:0.125333\n",
      "[459]\teval-error:0.284\ttrain-error:0.125333\n",
      "[460]\teval-error:0.284\ttrain-error:0.124\n",
      "[461]\teval-error:0.284\ttrain-error:0.125333\n",
      "[462]\teval-error:0.284\ttrain-error:0.125333\n",
      "[463]\teval-error:0.284\ttrain-error:0.125333\n",
      "[464]\teval-error:0.284\ttrain-error:0.125333\n",
      "[465]\teval-error:0.28\ttrain-error:0.125333\n",
      "[466]\teval-error:0.28\ttrain-error:0.125333\n",
      "[467]\teval-error:0.28\ttrain-error:0.125333\n",
      "[468]\teval-error:0.28\ttrain-error:0.125333\n",
      "[469]\teval-error:0.28\ttrain-error:0.125333\n",
      "[470]\teval-error:0.28\ttrain-error:0.125333\n",
      "[471]\teval-error:0.28\ttrain-error:0.125333\n",
      "[472]\teval-error:0.28\ttrain-error:0.125333\n",
      "[473]\teval-error:0.28\ttrain-error:0.125333\n",
      "[474]\teval-error:0.28\ttrain-error:0.124\n",
      "[475]\teval-error:0.28\ttrain-error:0.125333\n",
      "[476]\teval-error:0.28\ttrain-error:0.126667\n",
      "[477]\teval-error:0.28\ttrain-error:0.126667\n",
      "[478]\teval-error:0.28\ttrain-error:0.126667\n",
      "[479]\teval-error:0.28\ttrain-error:0.126667\n",
      "[480]\teval-error:0.28\ttrain-error:0.128\n",
      "[481]\teval-error:0.28\ttrain-error:0.126667\n",
      "[482]\teval-error:0.28\ttrain-error:0.126667\n",
      "[483]\teval-error:0.28\ttrain-error:0.125333\n",
      "[484]\teval-error:0.28\ttrain-error:0.126667\n",
      "[485]\teval-error:0.28\ttrain-error:0.126667\n",
      "[486]\teval-error:0.28\ttrain-error:0.125333\n",
      "[487]\teval-error:0.28\ttrain-error:0.125333\n",
      "[488]\teval-error:0.28\ttrain-error:0.126667\n",
      "[489]\teval-error:0.28\ttrain-error:0.126667\n",
      "[490]\teval-error:0.28\ttrain-error:0.125333\n",
      "[491]\teval-error:0.28\ttrain-error:0.125333\n",
      "[492]\teval-error:0.28\ttrain-error:0.126667\n",
      "[493]\teval-error:0.28\ttrain-error:0.125333\n",
      "[494]\teval-error:0.28\ttrain-error:0.125333\n",
      "[495]\teval-error:0.28\ttrain-error:0.125333\n",
      "[496]\teval-error:0.28\ttrain-error:0.125333\n",
      "[497]\teval-error:0.28\ttrain-error:0.124\n",
      "[498]\teval-error:0.28\ttrain-error:0.124\n",
      "[499]\teval-error:0.28\ttrain-error:0.124\n",
      "[500]\teval-error:0.28\ttrain-error:0.124\n",
      "[501]\teval-error:0.28\ttrain-error:0.124\n",
      "[502]\teval-error:0.28\ttrain-error:0.124\n",
      "[503]\teval-error:0.28\ttrain-error:0.124\n",
      "[504]\teval-error:0.28\ttrain-error:0.124\n",
      "[505]\teval-error:0.28\ttrain-error:0.124\n",
      "[506]\teval-error:0.28\ttrain-error:0.124\n",
      "[507]\teval-error:0.28\ttrain-error:0.124\n",
      "[508]\teval-error:0.28\ttrain-error:0.124\n",
      "[509]\teval-error:0.28\ttrain-error:0.122667\n",
      "[510]\teval-error:0.28\ttrain-error:0.125333\n",
      "[511]\teval-error:0.28\ttrain-error:0.125333\n",
      "[512]\teval-error:0.28\ttrain-error:0.124\n",
      "[513]\teval-error:0.28\ttrain-error:0.126667\n",
      "[514]\teval-error:0.28\ttrain-error:0.125333\n",
      "[515]\teval-error:0.28\ttrain-error:0.124\n",
      "[516]\teval-error:0.28\ttrain-error:0.125333\n",
      "[517]\teval-error:0.28\ttrain-error:0.122667\n",
      "[518]\teval-error:0.28\ttrain-error:0.122667\n",
      "[519]\teval-error:0.28\ttrain-error:0.122667\n",
      "[520]\teval-error:0.28\ttrain-error:0.122667\n",
      "[521]\teval-error:0.28\ttrain-error:0.122667\n",
      "[522]\teval-error:0.28\ttrain-error:0.122667\n",
      "[523]\teval-error:0.28\ttrain-error:0.122667\n",
      "[524]\teval-error:0.28\ttrain-error:0.122667\n",
      "[525]\teval-error:0.28\ttrain-error:0.124\n",
      "[526]\teval-error:0.28\ttrain-error:0.122667\n",
      "[527]\teval-error:0.28\ttrain-error:0.122667\n",
      "[528]\teval-error:0.28\ttrain-error:0.122667\n",
      "[529]\teval-error:0.28\ttrain-error:0.124\n",
      "[530]\teval-error:0.28\ttrain-error:0.122667\n",
      "[531]\teval-error:0.28\ttrain-error:0.124\n",
      "[532]\teval-error:0.28\ttrain-error:0.125333\n",
      "[533]\teval-error:0.28\ttrain-error:0.125333\n",
      "[534]\teval-error:0.28\ttrain-error:0.125333\n",
      "[535]\teval-error:0.28\ttrain-error:0.126667\n",
      "[536]\teval-error:0.28\ttrain-error:0.125333\n",
      "[537]\teval-error:0.28\ttrain-error:0.125333\n",
      "[538]\teval-error:0.28\ttrain-error:0.126667\n",
      "[539]\teval-error:0.28\ttrain-error:0.125333\n",
      "[540]\teval-error:0.28\ttrain-error:0.125333\n",
      "[541]\teval-error:0.28\ttrain-error:0.125333\n",
      "[542]\teval-error:0.28\ttrain-error:0.125333\n",
      "[543]\teval-error:0.28\ttrain-error:0.125333\n",
      "[544]\teval-error:0.28\ttrain-error:0.125333\n",
      "[545]\teval-error:0.28\ttrain-error:0.125333\n",
      "[546]\teval-error:0.28\ttrain-error:0.126667\n",
      "[547]\teval-error:0.28\ttrain-error:0.126667\n",
      "[548]\teval-error:0.28\ttrain-error:0.125333\n",
      "[549]\teval-error:0.28\ttrain-error:0.126667\n",
      "[550]\teval-error:0.28\ttrain-error:0.126667\n",
      "[551]\teval-error:0.28\ttrain-error:0.128\n",
      "[552]\teval-error:0.28\ttrain-error:0.128\n",
      "[553]\teval-error:0.28\ttrain-error:0.126667\n",
      "[554]\teval-error:0.28\ttrain-error:0.128\n",
      "[555]\teval-error:0.28\ttrain-error:0.126667\n",
      "[556]\teval-error:0.28\ttrain-error:0.128\n",
      "[557]\teval-error:0.28\ttrain-error:0.128\n",
      "[558]\teval-error:0.28\ttrain-error:0.128\n",
      "[559]\teval-error:0.28\ttrain-error:0.126667\n",
      "[560]\teval-error:0.28\ttrain-error:0.128\n",
      "[561]\teval-error:0.28\ttrain-error:0.128\n",
      "[562]\teval-error:0.28\ttrain-error:0.128\n",
      "[563]\teval-error:0.28\ttrain-error:0.128\n",
      "[564]\teval-error:0.28\ttrain-error:0.128\n",
      "[565]\teval-error:0.28\ttrain-error:0.128\n",
      "[566]\teval-error:0.28\ttrain-error:0.128\n",
      "[567]\teval-error:0.28\ttrain-error:0.128\n",
      "[568]\teval-error:0.28\ttrain-error:0.128\n",
      "[569]\teval-error:0.28\ttrain-error:0.128\n",
      "[570]\teval-error:0.28\ttrain-error:0.128\n",
      "[571]\teval-error:0.28\ttrain-error:0.128\n",
      "[572]\teval-error:0.28\ttrain-error:0.128\n",
      "[573]\teval-error:0.28\ttrain-error:0.128\n",
      "[574]\teval-error:0.28\ttrain-error:0.128\n",
      "[575]\teval-error:0.28\ttrain-error:0.128\n",
      "[576]\teval-error:0.28\ttrain-error:0.128\n",
      "[577]\teval-error:0.28\ttrain-error:0.128\n",
      "[578]\teval-error:0.28\ttrain-error:0.128\n",
      "[579]\teval-error:0.28\ttrain-error:0.128\n",
      "[580]\teval-error:0.28\ttrain-error:0.128\n",
      "[581]\teval-error:0.28\ttrain-error:0.128\n",
      "[582]\teval-error:0.28\ttrain-error:0.128\n",
      "[583]\teval-error:0.28\ttrain-error:0.129333\n",
      "[584]\teval-error:0.28\ttrain-error:0.128\n",
      "[585]\teval-error:0.28\ttrain-error:0.128\n",
      "[586]\teval-error:0.28\ttrain-error:0.129333\n",
      "[587]\teval-error:0.28\ttrain-error:0.128\n",
      "[588]\teval-error:0.28\ttrain-error:0.128\n",
      "[589]\teval-error:0.28\ttrain-error:0.129333\n",
      "[590]\teval-error:0.28\ttrain-error:0.128\n",
      "[591]\teval-error:0.28\ttrain-error:0.128\n",
      "[592]\teval-error:0.28\ttrain-error:0.128\n",
      "[593]\teval-error:0.28\ttrain-error:0.128\n",
      "[594]\teval-error:0.28\ttrain-error:0.128\n",
      "[595]\teval-error:0.28\ttrain-error:0.128\n",
      "[596]\teval-error:0.28\ttrain-error:0.129333\n",
      "[597]\teval-error:0.28\ttrain-error:0.128\n",
      "[598]\teval-error:0.28\ttrain-error:0.128\n",
      "[599]\teval-error:0.28\ttrain-error:0.128\n",
      "[600]\teval-error:0.28\ttrain-error:0.129333\n",
      "[601]\teval-error:0.28\ttrain-error:0.128\n",
      "[602]\teval-error:0.28\ttrain-error:0.128\n",
      "[603]\teval-error:0.28\ttrain-error:0.129333\n",
      "[604]\teval-error:0.28\ttrain-error:0.129333\n",
      "[605]\teval-error:0.28\ttrain-error:0.128\n",
      "[606]\teval-error:0.28\ttrain-error:0.128\n",
      "[607]\teval-error:0.28\ttrain-error:0.128\n",
      "[608]\teval-error:0.28\ttrain-error:0.129333\n",
      "[609]\teval-error:0.28\ttrain-error:0.128\n",
      "Stopping. Best iteration:\n",
      "[509]\teval-error:0.28\ttrain-error:0.122667\n",
      "\n",
      "[0]\teval-error:0.288\ttrain-error:0.174667\n",
      "Multiple eval metrics have been passed: 'train-error' will be used for early stopping.\n",
      "\n",
      "Will train until train-error hasn't improved in 100 rounds.\n",
      "[1]\teval-error:0.296\ttrain-error:0.164\n",
      "[2]\teval-error:0.296\ttrain-error:0.165333\n",
      "[3]\teval-error:0.292\ttrain-error:0.166667\n",
      "[4]\teval-error:0.292\ttrain-error:0.16\n",
      "[5]\teval-error:0.288\ttrain-error:0.161333\n",
      "[6]\teval-error:0.292\ttrain-error:0.164\n",
      "[7]\teval-error:0.292\ttrain-error:0.156\n",
      "[8]\teval-error:0.288\ttrain-error:0.162667\n",
      "[9]\teval-error:0.288\ttrain-error:0.164\n",
      "[10]\teval-error:0.296\ttrain-error:0.165333\n",
      "[11]\teval-error:0.3\ttrain-error:0.162667\n",
      "[12]\teval-error:0.3\ttrain-error:0.162667\n",
      "[13]\teval-error:0.296\ttrain-error:0.164\n",
      "[14]\teval-error:0.3\ttrain-error:0.16\n",
      "[15]\teval-error:0.304\ttrain-error:0.161333\n",
      "[16]\teval-error:0.304\ttrain-error:0.158667\n",
      "[17]\teval-error:0.304\ttrain-error:0.157333\n",
      "[18]\teval-error:0.308\ttrain-error:0.154667\n",
      "[19]\teval-error:0.304\ttrain-error:0.153333\n",
      "[20]\teval-error:0.296\ttrain-error:0.152\n",
      "[21]\teval-error:0.3\ttrain-error:0.150667\n",
      "[22]\teval-error:0.3\ttrain-error:0.150667\n",
      "[23]\teval-error:0.296\ttrain-error:0.152\n",
      "[24]\teval-error:0.296\ttrain-error:0.149333\n",
      "[25]\teval-error:0.296\ttrain-error:0.149333\n",
      "[26]\teval-error:0.292\ttrain-error:0.150667\n",
      "[27]\teval-error:0.292\ttrain-error:0.150667\n",
      "[28]\teval-error:0.292\ttrain-error:0.146667\n",
      "[29]\teval-error:0.292\ttrain-error:0.146667\n",
      "[30]\teval-error:0.28\ttrain-error:0.144\n",
      "[31]\teval-error:0.292\ttrain-error:0.145333\n",
      "[32]\teval-error:0.288\ttrain-error:0.141333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[33]\teval-error:0.288\ttrain-error:0.141333\n",
      "[34]\teval-error:0.284\ttrain-error:0.138667\n",
      "[35]\teval-error:0.28\ttrain-error:0.138667\n",
      "[36]\teval-error:0.28\ttrain-error:0.138667\n",
      "[37]\teval-error:0.28\ttrain-error:0.137333\n",
      "[38]\teval-error:0.276\ttrain-error:0.137333\n",
      "[39]\teval-error:0.28\ttrain-error:0.138667\n",
      "[40]\teval-error:0.276\ttrain-error:0.137333\n",
      "[41]\teval-error:0.276\ttrain-error:0.136\n",
      "[42]\teval-error:0.276\ttrain-error:0.136\n",
      "[43]\teval-error:0.272\ttrain-error:0.136\n",
      "[44]\teval-error:0.272\ttrain-error:0.137333\n",
      "[45]\teval-error:0.272\ttrain-error:0.136\n",
      "[46]\teval-error:0.272\ttrain-error:0.137333\n",
      "[47]\teval-error:0.272\ttrain-error:0.137333\n",
      "[48]\teval-error:0.272\ttrain-error:0.137333\n",
      "[49]\teval-error:0.272\ttrain-error:0.138667\n",
      "[50]\teval-error:0.272\ttrain-error:0.138667\n",
      "[51]\teval-error:0.272\ttrain-error:0.14\n",
      "[52]\teval-error:0.268\ttrain-error:0.137333\n",
      "[53]\teval-error:0.268\ttrain-error:0.137333\n",
      "[54]\teval-error:0.268\ttrain-error:0.137333\n",
      "[55]\teval-error:0.276\ttrain-error:0.138667\n",
      "[56]\teval-error:0.276\ttrain-error:0.137333\n",
      "[57]\teval-error:0.276\ttrain-error:0.136\n",
      "[58]\teval-error:0.276\ttrain-error:0.136\n",
      "[59]\teval-error:0.28\ttrain-error:0.137333\n",
      "[60]\teval-error:0.28\ttrain-error:0.137333\n",
      "[61]\teval-error:0.28\ttrain-error:0.138667\n",
      "[62]\teval-error:0.28\ttrain-error:0.138667\n",
      "[63]\teval-error:0.28\ttrain-error:0.138667\n",
      "[64]\teval-error:0.28\ttrain-error:0.138667\n",
      "[65]\teval-error:0.28\ttrain-error:0.138667\n",
      "[66]\teval-error:0.28\ttrain-error:0.138667\n",
      "[67]\teval-error:0.28\ttrain-error:0.138667\n",
      "[68]\teval-error:0.28\ttrain-error:0.138667\n",
      "[69]\teval-error:0.28\ttrain-error:0.14\n",
      "[70]\teval-error:0.284\ttrain-error:0.137333\n",
      "[71]\teval-error:0.288\ttrain-error:0.136\n",
      "[72]\teval-error:0.288\ttrain-error:0.136\n",
      "[73]\teval-error:0.288\ttrain-error:0.134667\n",
      "[74]\teval-error:0.288\ttrain-error:0.133333\n",
      "[75]\teval-error:0.288\ttrain-error:0.133333\n",
      "[76]\teval-error:0.288\ttrain-error:0.132\n",
      "[77]\teval-error:0.284\ttrain-error:0.132\n",
      "[78]\teval-error:0.288\ttrain-error:0.132\n",
      "[79]\teval-error:0.288\ttrain-error:0.128\n",
      "[80]\teval-error:0.292\ttrain-error:0.128\n",
      "[81]\teval-error:0.288\ttrain-error:0.125333\n",
      "[82]\teval-error:0.284\ttrain-error:0.125333\n",
      "[83]\teval-error:0.284\ttrain-error:0.125333\n",
      "[84]\teval-error:0.284\ttrain-error:0.125333\n",
      "[85]\teval-error:0.284\ttrain-error:0.124\n",
      "[86]\teval-error:0.288\ttrain-error:0.128\n",
      "[87]\teval-error:0.28\ttrain-error:0.128\n",
      "[88]\teval-error:0.288\ttrain-error:0.126667\n",
      "[89]\teval-error:0.288\ttrain-error:0.126667\n",
      "[90]\teval-error:0.288\ttrain-error:0.125333\n",
      "[91]\teval-error:0.292\ttrain-error:0.128\n",
      "[92]\teval-error:0.288\ttrain-error:0.126667\n",
      "[93]\teval-error:0.288\ttrain-error:0.128\n",
      "[94]\teval-error:0.284\ttrain-error:0.130667\n",
      "[95]\teval-error:0.284\ttrain-error:0.124\n",
      "[96]\teval-error:0.284\ttrain-error:0.124\n",
      "[97]\teval-error:0.292\ttrain-error:0.122667\n",
      "[98]\teval-error:0.292\ttrain-error:0.126667\n",
      "[99]\teval-error:0.284\ttrain-error:0.124\n",
      "[100]\teval-error:0.288\ttrain-error:0.122667\n",
      "[101]\teval-error:0.288\ttrain-error:0.122667\n",
      "[102]\teval-error:0.296\ttrain-error:0.122667\n",
      "[103]\teval-error:0.292\ttrain-error:0.122667\n",
      "[104]\teval-error:0.292\ttrain-error:0.121333\n",
      "[105]\teval-error:0.284\ttrain-error:0.121333\n",
      "[106]\teval-error:0.284\ttrain-error:0.121333\n",
      "[107]\teval-error:0.284\ttrain-error:0.121333\n",
      "[108]\teval-error:0.28\ttrain-error:0.121333\n",
      "[109]\teval-error:0.284\ttrain-error:0.121333\n",
      "[110]\teval-error:0.292\ttrain-error:0.122667\n",
      "[111]\teval-error:0.284\ttrain-error:0.121333\n",
      "[112]\teval-error:0.284\ttrain-error:0.122667\n",
      "[113]\teval-error:0.284\ttrain-error:0.118667\n",
      "[114]\teval-error:0.288\ttrain-error:0.118667\n",
      "[115]\teval-error:0.288\ttrain-error:0.12\n",
      "[116]\teval-error:0.284\ttrain-error:0.12\n",
      "[117]\teval-error:0.28\ttrain-error:0.12\n",
      "[118]\teval-error:0.272\ttrain-error:0.118667\n",
      "[119]\teval-error:0.276\ttrain-error:0.12\n",
      "[120]\teval-error:0.276\ttrain-error:0.117333\n",
      "[121]\teval-error:0.28\ttrain-error:0.12\n",
      "[122]\teval-error:0.276\ttrain-error:0.118667\n",
      "[123]\teval-error:0.276\ttrain-error:0.121333\n",
      "[124]\teval-error:0.276\ttrain-error:0.118667\n",
      "[125]\teval-error:0.276\ttrain-error:0.121333\n",
      "[126]\teval-error:0.28\ttrain-error:0.121333\n",
      "[127]\teval-error:0.276\ttrain-error:0.122667\n",
      "[128]\teval-error:0.276\ttrain-error:0.121333\n",
      "[129]\teval-error:0.276\ttrain-error:0.122667\n",
      "[130]\teval-error:0.28\ttrain-error:0.121333\n",
      "[131]\teval-error:0.28\ttrain-error:0.118667\n",
      "[132]\teval-error:0.28\ttrain-error:0.121333\n",
      "[133]\teval-error:0.276\ttrain-error:0.122667\n",
      "[134]\teval-error:0.272\ttrain-error:0.12\n",
      "[135]\teval-error:0.276\ttrain-error:0.117333\n",
      "[136]\teval-error:0.276\ttrain-error:0.118667\n",
      "[137]\teval-error:0.28\ttrain-error:0.114667\n",
      "[138]\teval-error:0.28\ttrain-error:0.114667\n",
      "[139]\teval-error:0.28\ttrain-error:0.114667\n",
      "[140]\teval-error:0.28\ttrain-error:0.113333\n",
      "[141]\teval-error:0.28\ttrain-error:0.114667\n",
      "[142]\teval-error:0.276\ttrain-error:0.116\n",
      "[143]\teval-error:0.276\ttrain-error:0.114667\n",
      "[144]\teval-error:0.276\ttrain-error:0.114667\n",
      "[145]\teval-error:0.272\ttrain-error:0.116\n",
      "[146]\teval-error:0.272\ttrain-error:0.118667\n",
      "[147]\teval-error:0.272\ttrain-error:0.117333\n",
      "[148]\teval-error:0.268\ttrain-error:0.117333\n",
      "[149]\teval-error:0.272\ttrain-error:0.117333\n",
      "[150]\teval-error:0.272\ttrain-error:0.117333\n",
      "[151]\teval-error:0.272\ttrain-error:0.117333\n",
      "[152]\teval-error:0.272\ttrain-error:0.117333\n",
      "[153]\teval-error:0.272\ttrain-error:0.118667\n",
      "[154]\teval-error:0.272\ttrain-error:0.117333\n",
      "[155]\teval-error:0.268\ttrain-error:0.118667\n",
      "[156]\teval-error:0.268\ttrain-error:0.117333\n",
      "[157]\teval-error:0.272\ttrain-error:0.117333\n",
      "[158]\teval-error:0.272\ttrain-error:0.117333\n",
      "[159]\teval-error:0.272\ttrain-error:0.117333\n",
      "[160]\teval-error:0.272\ttrain-error:0.118667\n",
      "[161]\teval-error:0.272\ttrain-error:0.117333\n",
      "[162]\teval-error:0.272\ttrain-error:0.118667\n",
      "[163]\teval-error:0.268\ttrain-error:0.118667\n",
      "[164]\teval-error:0.272\ttrain-error:0.118667\n",
      "[165]\teval-error:0.272\ttrain-error:0.118667\n",
      "[166]\teval-error:0.272\ttrain-error:0.12\n",
      "[167]\teval-error:0.272\ttrain-error:0.12\n",
      "[168]\teval-error:0.268\ttrain-error:0.118667\n",
      "[169]\teval-error:0.268\ttrain-error:0.117333\n",
      "[170]\teval-error:0.268\ttrain-error:0.117333\n",
      "[171]\teval-error:0.268\ttrain-error:0.117333\n",
      "[172]\teval-error:0.272\ttrain-error:0.117333\n",
      "[173]\teval-error:0.272\ttrain-error:0.117333\n",
      "[174]\teval-error:0.272\ttrain-error:0.117333\n",
      "[175]\teval-error:0.272\ttrain-error:0.116\n",
      "[176]\teval-error:0.272\ttrain-error:0.117333\n",
      "[177]\teval-error:0.268\ttrain-error:0.117333\n",
      "[178]\teval-error:0.268\ttrain-error:0.117333\n",
      "[179]\teval-error:0.268\ttrain-error:0.117333\n",
      "[180]\teval-error:0.268\ttrain-error:0.117333\n",
      "[181]\teval-error:0.268\ttrain-error:0.117333\n",
      "[182]\teval-error:0.272\ttrain-error:0.117333\n",
      "[183]\teval-error:0.272\ttrain-error:0.117333\n",
      "[184]\teval-error:0.272\ttrain-error:0.118667\n",
      "[185]\teval-error:0.268\ttrain-error:0.12\n",
      "[186]\teval-error:0.268\ttrain-error:0.12\n",
      "[187]\teval-error:0.268\ttrain-error:0.118667\n",
      "[188]\teval-error:0.268\ttrain-error:0.118667\n",
      "[189]\teval-error:0.268\ttrain-error:0.118667\n",
      "[190]\teval-error:0.268\ttrain-error:0.118667\n",
      "[191]\teval-error:0.268\ttrain-error:0.118667\n",
      "[192]\teval-error:0.272\ttrain-error:0.12\n",
      "[193]\teval-error:0.272\ttrain-error:0.12\n",
      "[194]\teval-error:0.272\ttrain-error:0.12\n",
      "[195]\teval-error:0.268\ttrain-error:0.12\n",
      "[196]\teval-error:0.268\ttrain-error:0.12\n",
      "[197]\teval-error:0.268\ttrain-error:0.12\n",
      "[198]\teval-error:0.268\ttrain-error:0.12\n",
      "[199]\teval-error:0.268\ttrain-error:0.12\n",
      "[200]\teval-error:0.268\ttrain-error:0.118667\n",
      "[201]\teval-error:0.268\ttrain-error:0.118667\n",
      "[202]\teval-error:0.272\ttrain-error:0.118667\n",
      "[203]\teval-error:0.272\ttrain-error:0.118667\n",
      "[204]\teval-error:0.272\ttrain-error:0.117333\n",
      "[205]\teval-error:0.272\ttrain-error:0.117333\n",
      "[206]\teval-error:0.272\ttrain-error:0.117333\n",
      "[207]\teval-error:0.272\ttrain-error:0.117333\n",
      "[208]\teval-error:0.272\ttrain-error:0.117333\n",
      "[209]\teval-error:0.272\ttrain-error:0.117333\n",
      "[210]\teval-error:0.272\ttrain-error:0.117333\n",
      "[211]\teval-error:0.272\ttrain-error:0.117333\n",
      "[212]\teval-error:0.272\ttrain-error:0.117333\n",
      "[213]\teval-error:0.272\ttrain-error:0.117333\n",
      "[214]\teval-error:0.272\ttrain-error:0.117333\n",
      "[215]\teval-error:0.272\ttrain-error:0.117333\n",
      "[216]\teval-error:0.272\ttrain-error:0.118667\n",
      "[217]\teval-error:0.272\ttrain-error:0.118667\n",
      "[218]\teval-error:0.272\ttrain-error:0.117333\n",
      "[219]\teval-error:0.272\ttrain-error:0.117333\n",
      "[220]\teval-error:0.272\ttrain-error:0.117333\n",
      "[221]\teval-error:0.272\ttrain-error:0.117333\n",
      "[222]\teval-error:0.272\ttrain-error:0.117333\n",
      "[223]\teval-error:0.272\ttrain-error:0.117333\n",
      "[224]\teval-error:0.272\ttrain-error:0.117333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[225]\teval-error:0.264\ttrain-error:0.117333\n",
      "[226]\teval-error:0.264\ttrain-error:0.117333\n",
      "[227]\teval-error:0.264\ttrain-error:0.117333\n",
      "[228]\teval-error:0.264\ttrain-error:0.117333\n",
      "[229]\teval-error:0.264\ttrain-error:0.117333\n",
      "[230]\teval-error:0.264\ttrain-error:0.117333\n",
      "[231]\teval-error:0.264\ttrain-error:0.117333\n",
      "[232]\teval-error:0.264\ttrain-error:0.117333\n",
      "[233]\teval-error:0.264\ttrain-error:0.117333\n",
      "[234]\teval-error:0.264\ttrain-error:0.117333\n",
      "[235]\teval-error:0.264\ttrain-error:0.117333\n",
      "[236]\teval-error:0.264\ttrain-error:0.117333\n",
      "[237]\teval-error:0.264\ttrain-error:0.117333\n",
      "[238]\teval-error:0.264\ttrain-error:0.117333\n",
      "[239]\teval-error:0.264\ttrain-error:0.117333\n",
      "[240]\teval-error:0.264\ttrain-error:0.117333\n",
      "Stopping. Best iteration:\n",
      "[140]\teval-error:0.28\ttrain-error:0.113333\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for beta in betas:\n",
    "    def weightloss(preds,dtrain):\n",
    "        y = dtrain.get_label()\n",
    "        p = 1.0 / (1.0 + np.exp(-preds ))\n",
    "        grad = p * (beta + y - beta*y) - y\n",
    "        hess = p*(1-p)*(beta + y - beta*y)\n",
    "        return grad,hess\n",
    "    bst_weightloss= xgb.train(params,data_train,num_boost_round=2000,evals = watch_list,early_stopping_rounds = 100,\n",
    "                      obj=weightloss,maximize=False)\n",
    "    ypred = bst_weightloss.predict(data_test)\n",
    "    y_pred = (ypred >= 0.5)*1\n",
    "  \n",
    "    \n",
    "    auc_score.append(metrics.roc_auc_score(y_test,ypred))\n",
    "    acc_score.append(metrics.accuracy_score(y_test,y_pred))\n",
    "    recall_score.append(metrics.recall_score(y_test,y_pred))\n",
    "    prec_score.append(metrics.precision_score(y_test,y_pred))\n",
    "    f1_score.append(metrics.f1_score(y_test,y_pred))\n",
    "    beta_score.append(fbeta_score(y_test, y_pred, beta=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.49600000000000005,\n",
       " 0.51304347826086949,\n",
       " 0.54444444444444451,\n",
       " 0.52702702702702708,\n",
       " 0.51470588235294112]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "betas.insert(0,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame([auc_score,acc_score,recall_score,prec_score,f1_score,beta_score]).T\n",
    "result.columns = [\"auc_score\",\"acc_score\",\"recall_score\",\"prec_score\",\"f1_score\",\"beta_score\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auc_score</th>\n",
       "      <th>acc_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>prec_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>beta_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.731976</td>\n",
       "      <td>0.748</td>\n",
       "      <td>0.430556</td>\n",
       "      <td>0.584906</td>\n",
       "      <td>0.496000</td>\n",
       "      <td>0.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.713561</td>\n",
       "      <td>0.552</td>\n",
       "      <td>0.819444</td>\n",
       "      <td>0.373418</td>\n",
       "      <td>0.513043</td>\n",
       "      <td>0.661435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.721403</td>\n",
       "      <td>0.672</td>\n",
       "      <td>0.680556</td>\n",
       "      <td>0.453704</td>\n",
       "      <td>0.544444</td>\n",
       "      <td>0.618687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.723237</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>0.513158</td>\n",
       "      <td>0.527027</td>\n",
       "      <td>0.535714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.709699</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.486111</td>\n",
       "      <td>0.546875</td>\n",
       "      <td>0.514706</td>\n",
       "      <td>0.497159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   auc_score  acc_score  recall_score  prec_score  f1_score  beta_score\n",
       "0   0.731976      0.748      0.430556    0.584906  0.496000    0.454545\n",
       "1   0.713561      0.552      0.819444    0.373418  0.513043    0.661435\n",
       "2   0.721403      0.672      0.680556    0.453704  0.544444    0.618687\n",
       "3   0.723237      0.720      0.541667    0.513158  0.527027    0.535714\n",
       "4   0.709699      0.736      0.486111    0.546875  0.514706    0.497159"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "name": "auc_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.7319756554307117,
          0.7135611735330837,
          0.721402933832709,
          0.723236579275905,
          0.7096988139825219
         ]
        },
        {
         "name": "acc_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.748,
          0.552,
          0.672,
          0.72,
          0.736
         ]
        },
        {
         "name": "recall_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.4305555555555556,
          0.8194444444444444,
          0.6805555555555556,
          0.5416666666666666,
          0.4861111111111111
         ]
        },
        {
         "name": "prec_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.5849056603773585,
          0.37341772151898733,
          0.4537037037037037,
          0.5131578947368421,
          0.546875
         ]
        },
        {
         "name": "f1_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.49600000000000005,
          0.5130434782608695,
          0.5444444444444445,
          0.5270270270270271,
          0.5147058823529411
         ]
        },
        {
         "name": "beta_score",
         "type": "bar",
         "x": [
          0,
          0.1,
          0.2,
          0.3,
          0.4
         ],
         "y": [
          0.4545454545454546,
          0.6614349775784752,
          0.6186868686868688,
          0.5357142857142857,
          0.4971590909090909
         ]
        }
       ],
       "layout": {
        "title": "Target variable distribution",
        "xaxis": {
         "title": "Risk Variable"
        },
        "yaxis": {
         "title": "Count"
        }
       }
      },
      "text/html": [
       "<div id=\"770bebfa-0e5c-4b3c-bd10-9f11162672d8\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"770bebfa-0e5c-4b3c-bd10-9f11162672d8\", [{\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.7319756554307117, 0.7135611735330837, 0.721402933832709, 0.723236579275905, 0.7096988139825219], \"name\": \"auc_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.748, 0.552, 0.672, 0.72, 0.736], \"name\": \"acc_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.4305555555555556, 0.8194444444444444, 0.6805555555555556, 0.5416666666666666, 0.4861111111111111], \"name\": \"recall_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.5849056603773585, 0.37341772151898733, 0.4537037037037037, 0.5131578947368421, 0.546875], \"name\": \"prec_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.49600000000000005, 0.5130434782608695, 0.5444444444444445, 0.5270270270270271, 0.5147058823529411], \"name\": \"f1_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.4545454545454546, 0.6614349775784752, 0.6186868686868688, 0.5357142857142857, 0.4971590909090909], \"name\": \"beta_score\"}], {\"yaxis\": {\"title\": \"Count\"}, \"xaxis\": {\"title\": \"Risk Variable\"}, \"title\": \"Target variable distribution\"}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"770bebfa-0e5c-4b3c-bd10-9f11162672d8\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"770bebfa-0e5c-4b3c-bd10-9f11162672d8\", [{\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.7319756554307117, 0.7135611735330837, 0.721402933832709, 0.723236579275905, 0.7096988139825219], \"name\": \"auc_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.748, 0.552, 0.672, 0.72, 0.736], \"name\": \"acc_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.4305555555555556, 0.8194444444444444, 0.6805555555555556, 0.5416666666666666, 0.4861111111111111], \"name\": \"recall_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.5849056603773585, 0.37341772151898733, 0.4537037037037037, 0.5131578947368421, 0.546875], \"name\": \"prec_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.49600000000000005, 0.5130434782608695, 0.5444444444444445, 0.5270270270270271, 0.5147058823529411], \"name\": \"f1_score\"}, {\"type\": \"bar\", \"x\": [0, 0.1, 0.2, 0.3, 0.4], \"y\": [0.4545454545454546, 0.6614349775784752, 0.6186868686868688, 0.5357142857142857, 0.4971590909090909], \"name\": \"beta_score\"}], {\"yaxis\": {\"title\": \"Count\"}, \"xaxis\": {\"title\": \"Risk Variable\"}, \"title\": \"Target variable distribution\"}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# it's a library that we work with plotly\n",
    "import plotly.offline as py \n",
    "py.init_notebook_mode(connected=True) # this code, allow us to work with offline plotly version\n",
    "import plotly.graph_objs as go # it's like \"plt\" of matplot\n",
    "import plotly.tools as tls # It's useful to we get some tools of plotly\n",
    "import warnings # This library will be used to ignore some warnings\n",
    "from collections import Counter # To do counter of some features\n",
    "\n",
    "trace0 = go.Bar(\n",
    "            x = betas,\n",
    "            y = auc_score,\n",
    "            name='auc_score'\n",
    "    )\n",
    "\n",
    "trace1 = go.Bar(\n",
    "            x = betas ,\n",
    "            y = acc_score,\n",
    "            name='acc_score'\n",
    "    )\n",
    "trace2 = go.Bar(\n",
    "            x = betas ,\n",
    "            y = recall_score,\n",
    "            name='recall_score'\n",
    "    )\n",
    "trace3 = go.Bar(\n",
    "            x = betas ,\n",
    "            y = prec_score,\n",
    "            name='prec_score'\n",
    "    )\n",
    "trace4 = go.Bar(\n",
    "            x = betas ,\n",
    "            y = f1_score,\n",
    "            name='f1_score'\n",
    "    )\n",
    "\n",
    "trace5 = go.Bar(\n",
    "            x = betas ,\n",
    "            y = beta_score,\n",
    "            name='beta_score'\n",
    "    )\n",
    "data = [trace0, trace1,trace2,trace3,trace4,trace5]\n",
    "\n",
    "layout = go.Layout(\n",
    "    \n",
    ")\n",
    "\n",
    "layout = go.Layout(\n",
    "    yaxis=dict(\n",
    "        title='Count'\n",
    "    ),\n",
    "    xaxis=dict(\n",
    "        title='Risk Variable'\n",
    "    ),\n",
    "    title='Target variable distribution'\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "\n",
    "py.iplot(fig, filename='grouped-bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result.index= betas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "result.plot(kind='bar',figsize=(10,6.18))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "beta = 0.2\n",
    "def weightloss(preds,dtrain):\n",
    "    y = dtrain.get_label()\n",
    "    p = 1.0 / (1.0 + np.exp(-preds ))\n",
    "    grad = p * (beta + y - beta*y) - y\n",
    "    hess = p*(1-p)*(beta + y - beta*y)\n",
    "    return grad,hess\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    \"booster\":\"gbtree\",\n",
    "    \"objective\":\"binary:logistic\",\n",
    "    \"eta\":0.1,\n",
    "    \"max_depth\":10,\n",
    "    \"missing\":0,\n",
    "    \"seed\":0,\n",
    "    \"silent\":1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bst_weightloss= xgb.train(params,data_train,num_boost_round=2000,evals = watch_list,early_stopping_rounds = 100,\n",
    "                          obj=weightloss,maximize=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "lamda = 0.1\n",
    "def general_logloss(preds,dtrain):\n",
    "    labels = dtrain.get_label()\n",
    "    preds = 1.0 / (1.0 + np.exp(-preds))\n",
    "    grads = preds - labels + lamda * (preds - obj_series)\n",
    "    hess = preds * (1.0 - preds) + lamda *(preds *(1.0 -preds))\n",
    "    return grad,hess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
